pybind.cc 62.8 KB
Newer Older
1
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
2 3 4 5 6

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

7
http://www.apache.org/licenses/LICENSE-2.0
8 9 10 11 12 13

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
L
lgone2000 已提交
14
#include <Python.h>
C
chengduoZH 已提交
15 16
#include <algorithm>
#include <map>
S
sneaxiy 已提交
17
#include <memory>
C
chengduoZH 已提交
18 19 20 21 22
#include <mutex>  // NOLINT // for call_once
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>
23

Y
Yi Wang 已提交
24 25 26
#include "paddle/fluid/framework/executor.h"
#include "paddle/fluid/framework/feed_fetch_method.h"
#include "paddle/fluid/framework/framework.pb.h"
S
sneaxiy 已提交
27
#include "paddle/fluid/framework/garbage_collector.h"
28
#include "paddle/fluid/framework/ir/alloc_continuous_space_for_grad_pass.h"
29
#include "paddle/fluid/framework/ir/pass_builder.h"
Y
Yi Wang 已提交
30 31 32
#include "paddle/fluid/framework/lod_rank_table.h"
#include "paddle/fluid/framework/lod_tensor.h"
#include "paddle/fluid/framework/lod_tensor_array.h"
S
sneaxiy 已提交
33
#include "paddle/fluid/framework/op_info.h"
34
#include "paddle/fluid/framework/op_registry.h"
Y
Yu Yang 已提交
35
#include "paddle/fluid/framework/parallel_executor.h"
Y
Yi Wang 已提交
36
#include "paddle/fluid/framework/prune.h"
Y
Refine  
Yu Yang 已提交
37
#include "paddle/fluid/framework/reader.h"
S
sneaxiy 已提交
38
#include "paddle/fluid/framework/scope_pool.h"
Y
Yi Wang 已提交
39
#include "paddle/fluid/framework/selected_rows.h"
X
Xin Pan 已提交
40
#include "paddle/fluid/framework/version.h"
41
#include "paddle/fluid/imperative/layer.h"
M
minqiyang 已提交
42
#include "paddle/fluid/imperative/profiler.h"
Y
Refine  
Yu Yang 已提交
43
#include "paddle/fluid/memory/allocation/allocator_strategy.h"
44
#include "paddle/fluid/memory/allocation/legacy_allocator.h"
D
dzhwinter 已提交
45
#include "paddle/fluid/operators/activation_op.h"
S
sneaxiy 已提交
46
#include "paddle/fluid/operators/py_func_op.h"
S
sneaxiy 已提交
47
#include "paddle/fluid/operators/reader/lod_tensor_blocking_queue.h"
Y
Yu Yang 已提交
48
#include "paddle/fluid/platform/cpu_info.h"
Y
Yi Wang 已提交
49
#include "paddle/fluid/platform/enforce.h"
50
#include "paddle/fluid/platform/init.h"
Y
Yi Wang 已提交
51 52
#include "paddle/fluid/platform/place.h"
#include "paddle/fluid/platform/profiler.h"
W
Wang Guibao 已提交
53
#include "paddle/fluid/pybind/async_executor_py.h"
Y
Yi Wang 已提交
54
#include "paddle/fluid/pybind/const_value.h"
D
dongdaxiang 已提交
55
#include "paddle/fluid/pybind/data_set_py.h"
Y
Yi Wang 已提交
56
#include "paddle/fluid/pybind/exception.h"
D
dongdaxiang 已提交
57
#include "paddle/fluid/pybind/fleet_wrapper_py.h"
58
#include "paddle/fluid/pybind/imperative.h"
F
flame 已提交
59
#include "paddle/fluid/pybind/inference_api.h"
F
flame 已提交
60
#include "paddle/fluid/pybind/ir.h"
W
wopeizl 已提交
61
#ifndef _WIN32
D
dongdaxiang 已提交
62
#include "paddle/fluid/pybind/nccl_wrapper_py.h"
W
wopeizl 已提交
63
#endif
64 65
#include "paddle/fluid/pybind/protobuf.h"
#include "paddle/fluid/pybind/pybind.h"  // NOLINT
S
sneaxiy 已提交
66
#include "paddle/fluid/pybind/reader_py.h"
Y
Yu Yang 已提交
67
#include "paddle/fluid/pybind/recordio.h"
Y
Yi Wang 已提交
68
#include "paddle/fluid/pybind/tensor_py.h"
69
#include "paddle/fluid/string/to_string.h"
70

D
Dong Zhihong 已提交
71
#ifdef PADDLE_WITH_CUDA
P
peizhilin 已提交
72
#ifndef _WIN32
Y
Yi Wang 已提交
73
#include "paddle/fluid/operators/nccl/nccl_gpu_common.h"
P
peizhilin 已提交
74
#endif
Y
Yi Wang 已提交
75 76
#include "paddle/fluid/platform/cuda_profiler.h"
#include "paddle/fluid/platform/gpu_info.h"
D
Dong Zhihong 已提交
77 78
#endif

M
minqiyang 已提交
79 80
#include "pybind11/stl.h"

81 82 83 84
DEFINE_bool(reader_queue_speed_test_mode, false,
            "If set true, the queue.pop will only get data from queue but not "
            "remove the data from queue for speed testing");

Q
Qiao Longfei 已提交
85 86 87
// disable auto conversion to list in Python
PYBIND11_MAKE_OPAQUE(paddle::framework::LoDTensorArray);

88
namespace paddle {
89
namespace pybind {
90
bool IsCompiledWithCUDA() {
91
#ifndef PADDLE_WITH_CUDA
Q
qijun 已提交
92 93 94 95 96 97
  return false;
#else
  return true;
#endif
}

98 99 100 101 102 103 104 105
bool IsCompiledWithMKLDNN() {
#ifndef PADDLE_WITH_MKLDNN
  return false;
#else
  return true;
#endif
}

106 107 108 109 110 111 112 113
bool IsCompiledWithNGRAPH() {
#ifndef PADDLE_WITH_NGRAPH
  return false;
#else
  return true;
#endif
}

114
bool IsCompiledWithBrpc() {
115
#ifndef PADDLE_WITH_DISTRIBUTE
116 117
  return false;
#endif
118 119 120 121 122 123

#ifdef PADDLE_WITH_GRPC
  return false;
#endif

  return true;
124 125
}

Y
update  
Yancey1989 已提交
126
bool IsCompiledWithDIST() {
Y
Yancey1989 已提交
127
#ifdef PADDLE_WITH_DISTRIBUTE
Y
update  
Yancey1989 已提交
128 129 130 131 132 133
  return true;
#else
  return false;
#endif
}

S
sneaxiy 已提交
134 135 136 137 138 139 140 141 142 143
template <typename PlaceType1, typename PlaceType2>
static inline bool IsSamePlace(const PlaceType1 &p1, const PlaceType2 &p2) {
  return paddle::platform::Place(p1) == paddle::platform::Place(p2);
}

template <typename PlaceType>
static inline int PlaceIndex(const PlaceType &p) {
  return static_cast<int>(paddle::platform::Place(p).which());
}

144
PYBIND11_MODULE(core, m) {
Y
Yu Yang 已提交
145 146 147
  // Not used, just make sure cpu_info.cc is linked.
  paddle::platform::CpuTotalPhysicalMemory();

Y
Refine  
Yu Yang 已提交
148
  paddle::memory::allocation::UseAllocatorStrategyGFlag();
S
sneaxiy 已提交
149

150
  m.doc() = "C++ core of PaddlePaddle";
151

152 153 154 155
  // using framework in this function. Since it is inside a function, it will
  // not cause namespace pollution.
  using namespace paddle::framework;  // NOLINT

156
  BindException(&m);
Y
Yu Yang 已提交
157

S
sneaxiy 已提交
158
  m.def(
S
sneaxiy 已提交
159
      "_append_python_callable_object_and_return_id",
S
sneaxiy 已提交
160 161 162 163
      [](py::object py_obj) -> size_t {
        return paddle::operators::AppendPythonCallableObjectAndReturnId(py_obj);
      });

S
sneaxiy 已提交
164 165 166
  m.def("_get_use_default_grad_op_desc_maker_ops",
        [] { return OpInfoMap::Instance().GetUseDefaultGradOpDescMakerOps(); });

S
sneaxiy 已提交
167 168 169
  // NOTE(zjl): ctest would load environment variables at the beginning even
  // though we have not `import paddle.fluid as fluid`. So we add this API
  // to enable eager deletion mode in unittest.
S
sneaxiy 已提交
170
  m.def("_set_eager_deletion_mode", &paddle::framework::SetEagerDeletionMode);
S
sneaxiy 已提交
171

172
  m.def("_set_fuse_parameter_group_size",
173
        &paddle::framework::ir::SetFuseParameterGroupsSize);
174
  m.def("_set_fuse_parameter_memory_size",
175
        &paddle::framework::ir::SetFuseParameterMemorySize);
176

S
sneaxiy 已提交
177 178 179
  m.add_object("_cleanup",
               py::capsule([]() { ScopePool::Instance().Clear(); }));

180 181 182 183 184 185 186
  m.def("get_mem_usage", [](int device) {
    return memory::allocation::GPUMemMonitor.GetMemUsage(device);
  });

  m.def("print_mem_usage",
        []() { return memory::allocation::GPUMemMonitor.PrintMemUsage(); });

M
minqiyang 已提交
187
  m.def("start_imperative_gperf_profiler",
M
minqiyang 已提交
188 189
        []() { imperative::StartProfile(); });

M
minqiyang 已提交
190
  m.def("stop_imperative_gperf_profiler", []() { imperative::StopProfile(); });
M
minqiyang 已提交
191

M
minqiyang 已提交
192
  py::class_<imperative::VarBase>(m, "VarBase", R"DOC()DOC")
193 194 195 196 197 198 199 200
      .def(
          py::init<const std::string &, paddle::framework::proto::VarType::Type,
                   const std::vector<int64_t>, const paddle::platform::CPUPlace,
                   bool, bool>())
      .def(
          py::init<const std::string &, paddle::framework::proto::VarType::Type,
                   const std::vector<int64_t>,
                   const paddle::platform::CUDAPlace, bool, bool>())
201
      .def("_run_backward",
X
Xin Pan 已提交
202
           [](imperative::VarBase &self) { self.RunBackward(); })
M
minqiyang 已提交
203
      .def("_grad_name", &imperative::VarBase::GradName)
M
minqiyang 已提交
204
      .def("_grad_value", &imperative::VarBase::GradValue)
X
Xin Pan 已提交
205
      .def("_clear_gradient", &imperative::VarBase::ClearGradient)
M
minqiyang 已提交
206
      .def("_grad_ivar",
M
minqiyang 已提交
207
           [](const imperative::VarBase &self) { return self.grads_; },
M
minqiyang 已提交
208
           py::return_value_policy::reference)
M
minqiyang 已提交
209
      .def("_copy_to",
P
Paddle CI 已提交
210
           [](const imperative::VarBase &self, const platform::CPUPlace &place,
M
minqiyang 已提交
211 212 213 214 215
              bool blocking) {
             std::unique_ptr<imperative::VarBase> new_var =
                 self.NewVarBase(place, blocking);
             return new_var.release();
           },
P
Paddle CI 已提交
216
           py::return_value_policy::take_ownership)
M
minqiyang 已提交
217
      .def("_copy_to",
P
Paddle CI 已提交
218
           [](const imperative::VarBase &self, const platform::CUDAPlace &place,
M
minqiyang 已提交
219 220 221 222 223
              bool blocking) {
             std::unique_ptr<imperative::VarBase> new_var =
                 self.NewVarBase(place, blocking);
             return new_var.release();
           },
M
minqiyang 已提交
224
           py::return_value_policy::take_ownership)
M
minqiyang 已提交
225
      .def("value", [](const imperative::VarBase &self) { return self.var_; },
M
minqiyang 已提交
226
           py::return_value_policy::reference)
227 228 229
      .def_property("name", &imperative::VarBase::Name,
                    &imperative::VarBase::SetName)
      .def_property_readonly("shape", &imperative::VarBase::Shape)
M
minqiyang 已提交
230
      .def_property_readonly("dtype", &imperative::VarBase::DataType)
231 232 233 234
      .def_property("persistable", &imperative::VarBase::IsPersistable,
                    &imperative::VarBase::SetPersistable)
      .def_property("stop_gradient", &imperative::VarBase::IsStopGradient,
                    &imperative::VarBase::SetStopGradient);
235

236
  py::class_<imperative::OpBase, PyOpBase>(m, "OpBase", R"DOC()DOC")
237
      .def(py::init<const std::string &>())
238
      .def("register_backward_hooks",
239 240 241 242 243
           [](imperative::OpBase &self, const py::object &callable,
              bool front = false) {
             self.RegisterBackwardHooks(callable, front);
           },
           py::arg("callable"), py::arg("front") = false)
M
minqiyang 已提交
244 245 246 247 248 249 250 251 252 253
      .def_property("_trace_id",
                    [](const imperative::OpBase &self) {
                      pybind11::gil_scoped_release release;
                      return self.trace_id_;
                    },
                    [](imperative::OpBase &self, int trace_id) {
                      pybind11::gil_scoped_release release;
                      self.trace_id_ = trace_id;
                    },
                    py::return_value_policy::reference)
X
Xin Pan 已提交
254 255 256 257 258 259
      .def_property(
          "forward_id",
          [](const imperative::OpBase &self) { return self.forward_id_; },
          [](imperative::OpBase &self, int forward_id) {
            self.forward_id_ = forward_id;
          },
X
Xin Pan 已提交
260
          py::return_value_policy::reference)
X
polish  
Xin Pan 已提交
261
      .def_property_readonly("type", &imperative::OpBase::Type)
X
Xin Pan 已提交
262 263 264 265 266 267
      .def_property(
          "backward_id",
          [](const imperative::OpBase &self) { return self.backward_id_; },
          [](imperative::OpBase &self, int backward_id) {
            self.backward_id_ = backward_id;
          },
268 269
          py::return_value_policy::reference);

X
Xin Pan 已提交
270
  py::class_<imperative::Layer, Layer /* <--- trampoline*/> layer(m, "Layer");
271
  layer.def(py::init<>())
X
Xin Pan 已提交
272 273 274
      .def("forward", [](imperative::Layer &self,
                         const std::vector<imperative::VarBase> &inputs) {
        return self.Forward(inputs);
X
Xin Pan 已提交
275
      });
X
Xin Pan 已提交
276

X
polish  
Xin Pan 已提交
277
  py::class_<imperative::PyLayer>(m, "PyLayer")
X
Xin Pan 已提交
278
      .def(py::init<>())
X
Xin Pan 已提交
279 280
      .def_static(
          "apply",
X
Xin Pan 已提交
281
          [](int func_id, const std::vector<imperative::VarBase *> &inputs)
X
Xin Pan 已提交
282
              -> std::vector<imperative::VarBase *> {
283 284 285 286 287 288 289 290 291 292 293
                auto ret_vars = imperative::PyLayer::Apply(func_id, inputs);
                std::vector<imperative::VarBase *> outputs;
                outputs.reserve(ret_vars.size());
                for (size_t i = 0U; i != ret_vars.size(); ++i) {
                  framework::Variable *v = ret_vars[i];
                  // TODO(minqiyang): use unique_name generator to set a name
                  outputs.emplace_back(
                      new imperative::VarBase("", v, nullptr, true));
                }

                return outputs;
X
Xin Pan 已提交
294 295
              },
          py::return_value_policy::take_ownership)
X
polish  
Xin Pan 已提交
296 297 298 299 300
      .def_static("register_func",
                  [](int func_id, const py::object &callable) {
                    imperative::PyLayer::RegisterFunc(func_id, callable);
                  })
      .def_static("num_funcs", &imperative::PyLayer::NumFuncs);
X
Xin Pan 已提交
301

302
  BindImperative(&m);
303

304
  py::class_<Tensor>(m, "Tensor", py::buffer_protocol())
305
      .def("__array__", [](Tensor &self) { return TensorToPyArray(self); })
S
sneaxiy 已提交
306 307
      .def("_is_initialized",
           [](const Tensor &self) { return self.IsInitialized(); })
Y
yuyang18 已提交
308
      .def("_get_dims",
309
           [](const Tensor &self) { return vectorize(self.dims()); })
Y
yuyang18 已提交
310
      .def("_set_dims",
Q
qijun 已提交
311
           [](Tensor &self, const std::vector<int64_t> &dim) {
Y
Yu Yang 已提交
312
             self.Resize(make_ddim(dim));
Y
Yu Yang 已提交
313
           })
Y
yuyang18 已提交
314
      .def("_set_layout",
D
dzhwinter 已提交
315 316 317
           [](Tensor &self, const std::string &layout) {
             self.set_layout(StringToDataLayout(layout));
           })
Y
yuyang18 已提交
318
      .def("_alloc_float",
D
dzhwinter 已提交
319
           [](Tensor &self, paddle::platform::CUDAPlace &place) {
Q
qijun 已提交
320
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
321
           })
Y
yuyang18 已提交
322
      .def("_alloc_float",
Y
Yu Yang 已提交
323
           [](Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
324
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
325
           })
Y
yuyang18 已提交
326
      .def("_alloc_int",
Y
Yu Yang 已提交
327
           [](Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
328
             self.mutable_data<int>(place);
Y
Yu Yang 已提交
329
           })
Y
yuyang18 已提交
330
      .def("_alloc_int",
D
dzhwinter 已提交
331
           [](Tensor &self, paddle::platform::CUDAPlace &place) {
Q
qijun 已提交
332
             self.mutable_data<int>(place);
Q
qijun 已提交
333
           })
Y
yuyang18 已提交
334
      .def("_alloc_int",
C
chengduoZH 已提交
335 336 337
           [](Tensor &self, paddle::platform::CUDAPinnedPlace &place) {
             self.mutable_data<int>(place);
           })
Y
yuyang18 已提交
338
      .def("_alloc_float",
C
chengduoZH 已提交
339 340 341
           [](Tensor &self, paddle::platform::CUDAPinnedPlace &place) {
             self.mutable_data<float>(place);
           })
Y
Yu Yang 已提交
342 343
      .def("set", PyCPUTensorSetFromArray<float>)
      .def("set", PyCPUTensorSetFromArray<int>)
344
      .def("set", PyCPUTensorSetFromArray<double>)
345
      .def("set", PyCPUTensorSetFromArray<int64_t>)
Y
Yu Yang 已提交
346
      .def("set", PyCPUTensorSetFromArray<bool>)
347
      .def("set", PyCPUTensorSetFromArray<uint16_t>)
F
fengjiayi 已提交
348
      .def("set", PyCPUTensorSetFromArray<uint8_t>)
Q
qingqing01 已提交
349
      .def("set", PyCPUTensorSetFromArray<int8_t>)
350
#ifdef PADDLE_WITH_CUDA
Y
Yu Yang 已提交
351 352
      .def("set", PyCUDATensorSetFromArray<float>)
      .def("set", PyCUDATensorSetFromArray<int>)
353
      .def("set", PyCUDATensorSetFromArray<double>)
354
      .def("set", PyCUDATensorSetFromArray<int64_t>)
Y
Yu Yang 已提交
355
      .def("set", PyCUDATensorSetFromArray<bool>)
356
      .def("set", PyCUDATensorSetFromArray<uint16_t>)
F
fengjiayi 已提交
357
      .def("set", PyCUDATensorSetFromArray<uint8_t>)
Q
qingqing01 已提交
358
      .def("set", PyCUDATensorSetFromArray<int8_t>)
C
chengduoZH 已提交
359 360 361 362 363 364
      .def("set", PyCUDAPinnedTensorSetFromArray<float>)
      .def("set", PyCUDAPinnedTensorSetFromArray<int>)
      .def("set", PyCUDAPinnedTensorSetFromArray<double>)
      .def("set", PyCUDAPinnedTensorSetFromArray<int64_t>)
      .def("set", PyCUDAPinnedTensorSetFromArray<bool>)
      .def("set", PyCUDAPinnedTensorSetFromArray<uint16_t>)
F
fengjiayi 已提交
365
      .def("set", PyCUDAPinnedTensorSetFromArray<uint8_t>)
Q
qingqing01 已提交
366
      .def("set", PyCUDAPinnedTensorSetFromArray<int8_t>)
Q
qijun 已提交
367
#endif
368
      .def("shape", [](Tensor &self) { return vectorize(self.dims()); })
Y
yuyang18 已提交
369 370 371 372
      .def("_set_float_element", TensorSetElement<float>)
      .def("_get_float_element", TensorGetElement<float>)
      .def("_set_double_element", TensorSetElement<double>)
      .def("_get_double_element", TensorGetElement<double>)
X
xuezhong 已提交
373
      .def("_place", [](Tensor &self) { return self.place(); })
W
wopeizl 已提交
374 375
      .def("_dtype", [](Tensor &self) { return self.type(); })
      .def("__getitem__", PySliceTensor, py::return_value_policy::reference);
Y
Yu Yang 已提交
376

X
Xin Pan 已提交
377 378 379 380 381 382 383 384 385
  py::class_<LoDTensor, Tensor>(m, "LoDTensor", R"DOC(
    LoDTensor is a Tensor with optional LoD information.

    np.array(lod_tensor) can convert LoDTensor to numpy array.
    lod_tensor.lod() can retrieve the LoD information.

    LoD is short for Level of Details and is usually used for varied sequence
    length. You can skip the following comment if you don't need optional LoD.

Z
Zeng Jinle 已提交
386 387 388
    For example, a LoDTensor X can look like the example below. It contains 
    2 sequences. The first has length 2 and the second has length 3, as 
    described by x.lod.
X
Xin Pan 已提交
389

Z
Zeng Jinle 已提交
390 391 392
    The first tensor dimension 5=2+3 is calculated from LoD if it's available.
    It means the total number of sequence element. In X, each element has 2
    columns, hence [5, 2].
X
Xin Pan 已提交
393

Z
Zeng Jinle 已提交
394 395 396
    x.lod  = [[2, 3]]
     
    x.data = [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]]
X
Xin Pan 已提交
397

Z
Zeng Jinle 已提交
398
    x.shape = [5, 2]
X
Xin Pan 已提交
399

Z
Zeng Jinle 已提交
400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416
    LoD can have multiple levels (for example, a paragraph can have multiple
    sentences and a sentence can have multiple words). In the following
    LodTensor Y, the lod_level is 2. It means there are 2 sequence, the
    first sequence length is 2 (has 2 sub-sequences), the second one's
    length is 1. The first sequence's 2 sub-sequences have length 2 and 2,
    respectively. And the second sequence's 1 sub-sequence has length 3.

    y.lod = [[2 1], [2 2 3]]

    y.shape = [2+2+3, ...]

    Examples:
        .. code-block:: python

          import paddle.fluid as fluid

          t = fluid.LoDTensor()
X
Xin Pan 已提交
417 418 419 420 421 422 423 424 425 426 427 428

  Note:
      In above description, LoD is length-based. In Paddle internal
      implementation, lod is offset-based. Hence, internally,
      y.lod is represented as [[0, 2, 3], [0, 2, 4, 7]] (length-based
      equivlent would be [[2-0, 3-2], [2-0, 4-2, 7-4]]).

      Sometimes LoD is called recursive_sequence_length to be more
      self-explanatory. In this case, it must be length-based. Due to history
      reasons. when LoD is called lod in public API, it might be offset-based.
      Users should be careful about it.
        )DOC")
429
      .def("__array__", [](Tensor &self) { return TensorToPyArray(self); })
430 431 432 433 434 435 436 437 438 439 440 441 442 443
      .def("__init__",
           [](LoDTensor &instance, const std::vector<std::vector<size_t>>
                                       &recursive_sequence_lengths) {
             LoD new_lod;
             new_lod.reserve(recursive_sequence_lengths.size());
             std::copy(recursive_sequence_lengths.begin(),
                       recursive_sequence_lengths.end(),
                       std::back_inserter(new_lod));
             LoD new_offset_lod = ConvertToOffsetBasedLoD(new_lod);
             PADDLE_ENFORCE(
                 CheckLoD(new_offset_lod, -1),
                 "the provided recursive_sequence_lengths info is invalid");
             new (&instance) LoDTensor(new_offset_lod);
           })
Y
Yu Yang 已提交
444
      .def("__init__", [](LoDTensor &instance) { new (&instance) LoDTensor(); })
G
gongweibao 已提交
445 446 447 448 449
      // We implement offset based LOD in C++ while we use length based with
      // Python API. So we changed set_lod to set_recursive_sequence_lengths to
      // avoid misuse.
      // The discussion is here:
      // https://github.com/PaddlePaddle/Paddle/issues/10855
D
dangqingqing 已提交
450
      .def("set_lod",
451
           [](LoDTensor &self, const std::vector<std::vector<size_t>> &lod) {
452
             // the input lod is offset-based level-of-detail info
Y
Yu Yang 已提交
453
             LoD new_lod;
454 455
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
456 457
             PADDLE_ENFORCE(CheckLoD(new_lod, vectorize(self.dims()).front()),
                            "the provided lod info is invalid");
458
             self.set_lod(new_lod);
S
sneaxiy 已提交
459 460 461 462 463 464
           },
           py::arg("lod"), R"DOC(
           Set LoD of the LoDTensor.

           Args:
               lod (List[List[int]]): the lod to be set.
Z
Zeng Jinle 已提交
465 466 467 468 469 470 471 472 473 474

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_lod([[0, 2, 5]])
S
sneaxiy 已提交
475
           )DOC")
476 477 478 479 480 481 482 483 484 485 486 487 488 489 490
      .def("set_recursive_sequence_lengths",
           [](LoDTensor &self, const std::vector<std::vector<size_t>>
                                   &recursive_sequence_lengths) {
             // the input recursive_sequence_lengths is length-based
             // level-of-detail info
             LoD new_lod;
             new_lod.reserve(recursive_sequence_lengths.size());
             std::copy(recursive_sequence_lengths.begin(),
                       recursive_sequence_lengths.end(),
                       std::back_inserter(new_lod));
             LoD new_offset_lod = ConvertToOffsetBasedLoD(new_lod);
             PADDLE_ENFORCE(
                 CheckLoD(new_offset_lod, vectorize(self.dims()).front()),
                 "the provided recursive_sequence_lengths info is invalid");
             self.set_lod(new_offset_lod);
S
sneaxiy 已提交
491 492 493 494
           },
           py::arg("recursive_sequence_lengths"), R"DOC(
           Set LoD of the LoDTensor according to recursive sequence length.

S
sneaxiy 已提交
495
           For example, if recursive_sequence_lengths=[[2, 3]], meaning that
496 497
           there are two sequences with length 2 and 3 respectively, the
           corresponding lod would be [[0, 2, 2+3]], i.e, [[0, 2, 5]].
S
sneaxiy 已提交
498 499

           Args:
500
                recursive_sequence_lengths (List[List[int]]): sequence lengths.
Z
Zeng Jinle 已提交
501 502 503 504 505 506 507 508 509 510

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_recursive_sequence_lengths([[2, 3]])
S
sneaxiy 已提交
511
           )DOC")
512 513 514 515 516 517 518 519
      .def("lod",
           [](LoDTensor &self) -> std::vector<std::vector<size_t>> {
             // output the offset-based lod info
             LoD lod = self.lod();
             std::vector<std::vector<size_t>> new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             return new_lod;
S
sneaxiy 已提交
520 521 522 523 524 525
           },
           R"DOC(
           Return the LoD of the LoDTensor.

           Returns:
               out (List[List[int]]): the lod of the LoDTensor.
Z
Zeng Jinle 已提交
526 527 528 529 530 531 532 533 534 535 536

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_lod([[0, 2, 5]])
                 print(t.lod()) # [[0, 2, 5]]
S
sneaxiy 已提交
537
           )DOC")
G
gongweibao 已提交
538
      // Set above comments of set_lod.
539 540 541 542 543 544 545 546
      .def("recursive_sequence_lengths",
           [](LoDTensor &self) -> std::vector<std::vector<size_t>> {
             // output the length-based lod info
             LoD lod = ConvertToLengthBasedLoD(self.lod());
             std::vector<std::vector<size_t>> new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             return new_lod;
S
sneaxiy 已提交
547 548 549 550 551
           },
           R"DOC(
           Return the sequence length of the LoDTensor corresponding to LoD.

           Returns:
552
               out (List[List[int]): the sequence lengths.
Z
Zeng Jinle 已提交
553 554 555 556 557 558 559 560 561 562 563

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_recursive_sequence_lengths([[2, 3]])
                 print(t.recursive_sequence_lengths()) # [[2, 3]]
S
sneaxiy 已提交
564 565 566 567 568 569 570 571 572 573 574 575
           )DOC")
      .def("has_valid_recursive_sequence_lengths",
           [](LoDTensor &self) -> bool {
             // Check that the lod info is valid and match the outermost
             // dimension of the LoDTensor data
             return CheckLoD(self.lod(), vectorize(self.dims()).front());
           },
           R"DOC(
           Check whether the lod of the LoDTensor is valid.

           Returns:
               out (bool): whether the lod is valid.
Z
Zeng Jinle 已提交
576 577 578 579 580 581 582 583 584 585 586

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_recursive_sequence_lengths([[2, 3]])
                 print(t.has_valid_recursive_sequence_lengths()) # True
W
wopeizl 已提交
587 588 589 590 591 592 593
           )DOC")
      .def("__getitem__", PySliceTensor, py::return_value_policy::reference,
           R"DOC(
           Slice the original Tensor, and remove the LoD information.

           Returns:
               out (Tensor): new Tensor(NOT LoDTensor).
S
sneaxiy 已提交
594
           )DOC");
D
dangqingqing 已提交
595

Q
qijun 已提交
596 597 598 599 600 601 602 603 604 605 606
  py::class_<SelectedRows>(m, "SelectedRows")
      .def("__init__",
           [](SelectedRows &instance) { new (&instance) SelectedRows(); })
      .def("__init__",
           [](SelectedRows &instance, const std::vector<int64_t> rows,
              const int64_t &height) {
             new (&instance) SelectedRows(rows, height);
           })
      .def("get_tensor",
           [](SelectedRows &self) { return self.mutable_value(); },
           py::return_value_policy::reference)
607 608
      .def("numel",
           [](SelectedRows &self) -> int64_t { return self.value().numel(); })
Q
qijun 已提交
609 610
      .def("set_height", &SelectedRows::set_height)
      .def("height", &SelectedRows::height)
Q
qijun 已提交
611 612 613 614 615 616 617 618 619
      .def("set_rows",
           [](SelectedRows &self, std::vector<int64_t> rows) {
#ifndef PADDLE_WITH_CUDA
             self.set_rows(rows);
#else
        Vector<int64_t> new_rows(rows);
        self.set_rows(new_rows);
#endif
           })
620
      .def("sync_index", [](SelectedRows &instance) { instance.SyncIndex(); })
621
      .def("rows", [](SelectedRows &self) {
622 623 624 625 626
        auto rows = self.rows();
        std::vector<int64_t> new_rows;
        new_rows.reserve(rows.size());
        std::copy(rows.begin(), rows.end(), std::back_inserter(new_rows));
        return new_rows;
627
      });
Q
qijun 已提交
628

629
  py::class_<Variable>(m, "Variable", R"DOC(Variable Class.
630 631 632

All parameter, weight, gradient are variables in Paddle.
)DOC")
S
sneaxiy 已提交
633
      .def(py::init<>())
634
      .def("is_int", [](const Variable &var) { return var.IsType<int>(); })
635
      .def("set_int",
636 637
           [](Variable &var, int val) -> void { *var.GetMutable<int>() = val; })
      .def("get_int", [](const Variable &var) -> int { return var.Get<int>(); })
638 639 640 641 642 643 644
      .def("is_float", [](const Variable &var) { return var.IsType<float>(); })
      .def("set_float",
           [](Variable &var, float val) -> void {
             *var.GetMutable<float>() = val;
           })
      .def("get_float",
           [](const Variable &var) -> float { return var.Get<float>(); })
Y
Yu Yang 已提交
645
      .def("get_tensor",
646 647
           [](Variable &self) -> LoDTensor * {
             return self.GetMutable<LoDTensor>();
D
dangqingqing 已提交
648 649
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
650 651 652
      .def("get_lod_rank_table",
           [](Variable &self) { return self.GetMutable<LoDRankTable>(); },
           py::return_value_policy::reference)
Q
qijun 已提交
653 654 655 656 657
      .def("get_selected_rows",
           [](Variable &self) -> SelectedRows * {
             return self.GetMutable<SelectedRows>();
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
658 659 660
      .def("get_lod_tensor_array",
           [](Variable &self) { return self.GetMutable<LoDTensorArray>(); },
           py::return_value_policy::reference)
P
peizhilin 已提交
661
#if (defined(PADDLE_WITH_CUDA) && !defined(_WIN32))
D
Dong Zhihong 已提交
662 663 664 665 666
      .def("get_communicator",
           [](Variable &self) -> platform::Communicator * {
             return self.GetMutable<platform::Communicator>();
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
667
#endif
Y
Refine  
Yu Yang 已提交
668 669 670 671 672
      .def("get_reader",
           [](Variable &self) -> framework::ReaderHolder * {
             PADDLE_ENFORCE(self.IsType<framework::ReaderHolder>());
             return self.GetMutable<framework::ReaderHolder>();
           },
W
wopeizl 已提交
673
           py::return_value_policy::reference);
674

S
sneaxiy 已提交
675
  BindReader(&m);
Y
Refine  
Yu Yang 已提交
676

S
sneaxiy 已提交
677 678 679 680
  using LoDTensorBlockingQueue =
      ::paddle::operators::reader::LoDTensorBlockingQueue;
  using LoDTensorBlockingQueueHolder =
      ::paddle::operators::reader::LoDTensorBlockingQueueHolder;
S
sneaxiy 已提交
681

S
sneaxiy 已提交
682 683
  py::class_<LoDTensorBlockingQueue, std::shared_ptr<LoDTensorBlockingQueue>>(
      m, "LoDTensorBlockingQueue", "")
S
sneaxiy 已提交
684
      .def("push",
S
sneaxiy 已提交
685
           [](LoDTensorBlockingQueue &self,
S
sneaxiy 已提交
686
              const std::vector<framework::LoDTensor> &lod_tensor_vec) {
S
sneaxiy 已提交
687
             pybind11::gil_scoped_release release;
S
sneaxiy 已提交
688
             return self.Push(lod_tensor_vec);
S
sneaxiy 已提交
689
           })
S
sneaxiy 已提交
690 691 692 693
      .def("size", &LoDTensorBlockingQueue::Size)
      .def("capacity", &LoDTensorBlockingQueue::Cap)
      .def("close", &LoDTensorBlockingQueue::Close)
      .def("is_closed", &LoDTensorBlockingQueue::IsClosed);
S
sneaxiy 已提交
694

S
sneaxiy 已提交
695
  m.def("init_lod_tensor_blocking_queue",
Q
Qiao Longfei 已提交
696 697
        [](Variable &var,
           size_t capacity) -> std::shared_ptr<LoDTensorBlockingQueue> {
Q
Qiao Longfei 已提交
698
          VLOG(1) << "init_lod_tensor_blocking_queue";
Q
Qiao Longfei 已提交
699 700 701 702
          auto *holder = var.GetMutable<LoDTensorBlockingQueueHolder>();
          holder->InitOnce(capacity, FLAGS_reader_queue_speed_test_mode);
          return holder->GetQueue();
        },
S
sneaxiy 已提交
703
        py::return_value_policy::copy);
S
sneaxiy 已提交
704

S
sneaxiy 已提交
705
  py::class_<Scope>(m, "_Scope", R"DOC(
Q
Qiao Longfei 已提交
706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724
    Scope is an association of a name to Variable. All variables belong to Scope.

    Variables in a parent scope can be retrieved from local scope.

    You need to specify a scope to run a Net, i.e., `exe.Run(&scope)`.
    One net can run in different scopes and update different variable in the
    scope.

    You can create var in a scope and get it from the scope.

    Examples:
        .. code-block:: python

          # create tensor from a scope and set value to it.
          param = scope.var('Param').get_tensor()
          param_array = np.full((height, row_numel), 5.0).astype("float32")
          param.set(param_array, place)

        )DOC")
S
sneaxiy 已提交
725 726
      .def("_remove_from_pool",
           [](Scope &self) { ScopePool::Instance().Remove(&self); })
D
dongzhihong 已提交
727
      .def("var",
728
           [](Scope &self, const std::string &name) -> Variable * {
D
dongzhihong 已提交
729
             return self.Var(name);
Y
Yu Yang 已提交
730
           },
S
sneaxiy 已提交
731 732
           py::arg("name"),
           R"DOC(
733
           Find or create variable named :code:`name` in the current scope.
S
sneaxiy 已提交
734

735
           If the variable named :code:`name` does not exist in the
S
sneaxiy 已提交
736
           current scope, the variable would be created. Otherwise,
737
           return the existing variable.
S
sneaxiy 已提交
738 739

           Args:
740 741
               name (str): the variable name.

S
sneaxiy 已提交
742
           Returns:
743
               out (core.Variable): the found or created variable.
S
sneaxiy 已提交
744 745 746 747
           )DOC",
           py::return_value_policy::reference)
      .def("find_var", &Scope::FindVar, py::arg("name"),
           R"DOC(
748
           Find variable named :code:`name` in the current scope or
S
sneaxiy 已提交
749
           its parent scope. Return None if not found.
750

S
sneaxiy 已提交
751 752
           Args:
               name (str): the variable name.
753

S
sneaxiy 已提交
754
           Returns:
755
               out (core.Variable|None): the found variable or None.
S
sneaxiy 已提交
756
           )DOC",
757
           py::return_value_policy::reference)
758
      .def("new_scope", [](Scope &self) -> Scope * { return &self.NewScope(); },
S
sneaxiy 已提交
759 760 761 762 763 764
           R"DOC(
           Create a new sub-scope of the current scope.

           Returns:
               out (core._Scope): the created sub-scope.
           )DOC",
765
           py::return_value_policy::reference)
S
sneaxiy 已提交
766 767 768
      .def("drop_kids", &Scope::DropKids,
           R"DOC(
           Delete all sub-scopes of the current scope.
S
sneaxiy 已提交
769 770
           )DOC")
      .def("_kids", &Scope::kids);
771

S
sneaxiy 已提交
772 773 774 775 776 777
  m.def("Scope",
        []() -> Scope * {
          auto *s = new Scope();
          ScopePool::Instance().Insert(std::unique_ptr<Scope>(s));
          return s;
        },
S
sneaxiy 已提交
778 779
        R"DOC(
        Create a new scope.
780

S
sneaxiy 已提交
781 782 783
        Returns:
            out (core._Scope): the created scope.
        )DOC",
S
sneaxiy 已提交
784 785
        py::return_value_policy::reference);

Y
Yu Yang 已提交
786 787
  //! @note: Be careful! PyBind will return std::string as an unicode, not
  //! Python str. If you want a str object, you should cast them in Python.
Y
Yu Yang 已提交
788 789
  m.def("get_all_op_protos", []() -> std::vector<py::bytes> {
    std::vector<py::bytes> ret_values;
790 791 792 793 794 795 796 797 798 799
    for (auto &iter : OpInfoMap::Instance().map()) {
      auto &info = iter.second;
      if (info.HasOpProtoAndChecker()) {
        std::string str;
        PADDLE_ENFORCE(
            info.Proto().SerializeToString(&str),
            "Serialize OpProto Error. This could be a bug of Paddle.");
        ret_values.emplace_back(str);
      }
    }
Y
Yu Yang 已提交
800 801
    return ret_values;
  });
802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817
  m.def(
      "get_grad_op_desc", [](const OpDesc &op_desc,
                             const std::unordered_set<std::string> &no_grad_set,
                             const std::vector<BlockDesc *> &grad_sub_block) {
        std::unordered_map<std::string, std::string> grad_to_var;
        std::vector<std::unique_ptr<OpDesc>> grad_op_descs =
            framework::OpInfoMap::Instance()
                .Get(op_desc.Type())
                .GradOpMaker()(op_desc, no_grad_set, &grad_to_var,
                               grad_sub_block);
        std::vector<OpDesc *> grad_op_desc_ptrs(grad_op_descs.size());
        std::transform(grad_op_descs.begin(), grad_op_descs.end(),
                       grad_op_desc_ptrs.begin(),
                       [](std::unique_ptr<OpDesc> &p) { return p.release(); });
        return std::make_pair(grad_op_desc_ptrs, grad_to_var);
      });
Y
Yu Yang 已提交
818
  m.def("prune", [](const ProgramDesc &origin,
819
                    const std::vector<std::array<size_t, 2>> &targets) {
Y
Yu Yang 已提交
820
    ProgramDesc prog_with_targets(origin);
821
    for (const auto &t : targets) {
822
      prog_with_targets.MutableBlock(t[0])->Op(t[1])->SetIsTarget(true);
823
    }
824
    proto::ProgramDesc pruned_desc;
825
    Prune(*prog_with_targets.Proto(), &pruned_desc);
Y
Yu Yang 已提交
826
    return new ProgramDesc(pruned_desc);
827
  });
828 829 830 831
  m.def("empty_var_name",
        []() { return std::string(framework::kEmptyVarName); });
  m.def("grad_var_suffix",
        []() { return std::string(framework::kGradVarSuffix); });
832 833 834
  m.def_submodule(
       "var_names",
       "The module will return special predefined variable name in Paddle")
Y
Yi Wang 已提交
835 836
      .def("empty", []() { return kEmptyVarName; })
      .def("temp", []() { return kTempVarName; });
Q
qijun 已提交
837
  // clang-format off
Y
Yu Yang 已提交
838
  py::class_<paddle::platform::DeviceContext>(m, "DeviceContext")
Q
qijun 已提交
839 840
      .def_static("create",
                  [](paddle::platform::CPUPlace& place)
Q
qijun 已提交
841
                      -> paddle::platform::DeviceContext* {
Q
qijun 已提交
842 843 844
                    return new paddle::platform::CPUDeviceContext();
                  })
      .def_static("create",
D
dzhwinter 已提交
845
                  [](paddle::platform::CUDAPlace& place)
Q
qijun 已提交
846
                      -> paddle::platform::DeviceContext* {
847
#ifndef PADDLE_WITH_CUDA
D
dzhwinter 已提交
848
                    PADDLE_THROW("CUDAPlace is not supported in CPU device.");
Q
qijun 已提交
849
#else
Q
qijun 已提交
850
                    return new paddle::platform::CUDADeviceContext(place);
Q
qijun 已提交
851
#endif
C
chengduoZH 已提交
852 853 854 855 856 857 858 859 860 861 862
                  })
          .def_static("create",
                [](paddle::platform::CUDAPinnedPlace& place)
                        -> paddle::platform::DeviceContext* {
#ifndef PADDLE_WITH_CUDA
                  PADDLE_THROW(
                        "CUDAPinnedPlace is not supported in CPU device.");
#else
                  return new paddle::platform::CUDAPinnedDeviceContext(place);
#endif
                });;
D
Dong Zhihong 已提交
863
// clang-format on
P
peizhilin 已提交
864
#if (defined(PADDLE_WITH_CUDA) && !defined(_WIN32))
D
Dong Zhihong 已提交
865 866
  py::class_<platform::Communicator>(m, "Communicator").def(py::init<>());
#endif
867 868 869 870
  py::class_<platform::CUDAPlace>(m, "CUDAPlace", R"DOC(
    CUDAPlace is a descriptor of a device. It represents a GPU, and each CUDAPlace
    has a dev_id to indicate the number of cards represented by the current CUDAPlace.
    The memory of CUDAPlace with different dev_id is not accessible.
L
lujun 已提交
871 872 873 874 875 876

    Examples:
        .. code-block:: python

          gpu_place = fluid.CUDAPlace(0)

877
        )DOC")
S
sneaxiy 已提交
878 879 880 881 882 883 884 885 886 887 888 889
      .def("__init__",
           [](platform::CUDAPlace &self, int dev_id) {
#ifdef PADDLE_WITH_CUDA
             PADDLE_ENFORCE(
                 dev_id >= 0 && dev_id < platform::GetCUDADeviceCount(),
                 "Invalid CUDAPlace(%d), must inside [0, %d)", dev_id,
                 platform::GetCUDADeviceCount());
             new (&self) platform::CUDAPlace(dev_id);
#else
             PADDLE_THROW("Cannot use CUDAPlace in CPU only version");
#endif
           })
S
sneaxiy 已提交
890 891 892 893 894 895
      .def("_type", &PlaceIndex<platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::Place>)
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::CPUPlace>)
      .def("_equals",
           &IsSamePlace<platform::CUDAPlace, platform::CUDAPinnedPlace>)
D
dzhwinter 已提交
896
      .def("__str__", string::to_string<const platform::CUDAPlace &>);
Q
qijun 已提交
897

898 899 900
  py::class_<paddle::platform::CPUPlace>(m, "CPUPlace", R"DOC(
    CPUPlace is a descriptor of a device. It represents a CPU, and the memory
    CPUPlace can be accessed by CPU.
L
lujun 已提交
901 902 903 904 905 906

    Examples:
        .. code-block:: python

          cpu_place = fluid.CPUPlace()

907
        )DOC")
908
      .def(py::init<>())
S
sneaxiy 已提交
909 910 911 912 913 914
      .def("_type", &PlaceIndex<platform::CPUPlace>)
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::Place>)
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::CPUPlace>)
      .def("_equals",
           &IsSamePlace<platform::CPUPlace, platform::CUDAPinnedPlace>)
915
      .def("__str__", string::to_string<const platform::CPUPlace &>);
Y
Yu Yang 已提交
916

917 918 919
  py::class_<paddle::platform::CUDAPinnedPlace>(m, "CUDAPinnedPlace", R"DOC(
    CUDAPinnedPlace is a descriptor of a device. The memory of CUDAPinnedPlace
    can be accessed by GPU and CPU.
L
lujun 已提交
920 921 922 923 924 925

    Examples:
        .. code-block:: python

          place = fluid.CUDAPinnedPlace()

926
        )DOC")
S
sneaxiy 已提交
927
      .def("__init__",
S
sneaxiy 已提交
928
           [](platform::CUDAPinnedPlace &self) {
S
sneaxiy 已提交
929 930 931
#ifndef PADDLE_WITH_CUDA
             PADDLE_THROW("Cannot use CUDAPinnedPlace in CPU only version");
#endif
S
sneaxiy 已提交
932
             new (&self) platform::CUDAPinnedPlace();
S
sneaxiy 已提交
933
           })
S
sneaxiy 已提交
934 935 936 937 938 939 940 941
      .def("_type", &PlaceIndex<platform::CUDAPinnedPlace>)
      .def("_equals", &IsSamePlace<platform::CUDAPinnedPlace, platform::Place>)
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::CUDAPlace>)
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::CPUPlace>)
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::CUDAPinnedPlace>)
C
chengduoZH 已提交
942 943
      .def("__str__", string::to_string<const platform::CUDAPinnedPlace &>);

Y
Yu Yang 已提交
944 945
  py::class_<platform::Place>(m, "Place")
      .def(py::init<>())
S
sneaxiy 已提交
946 947 948 949 950
      .def("_type", &PlaceIndex<platform::Place>)
      .def("_equals", &IsSamePlace<platform::Place, platform::Place>)
      .def("_equals", &IsSamePlace<platform::Place, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::Place, platform::CPUPlace>)
      .def("_equals", &IsSamePlace<platform::Place, platform::CUDAPinnedPlace>)
X
xuezhong 已提交
951 952
      .def("is_gpu_place",
           [](platform::Place &self) { return platform::is_gpu_place(self); })
S
sneaxiy 已提交
953 954 955 956 957 958
      .def("is_cpu_place",
           [](platform::Place &self) { return platform::is_cpu_place(self); })
      .def("is_cuda_pinned_place",
           [](platform::Place &self) {
             return platform::is_cuda_pinned_place(self);
           })
X
xuezhong 已提交
959 960 961 962
      .def("gpu_device_id",
           [](platform::Place &self) {
             return boost::get<platform::CUDAPlace>(self).device;
           })
S
sneaxiy 已提交
963 964
      .def("set_place", [](platform::Place &self,
                           const platform::Place &other) { self = other; })
Y
Yu Yang 已提交
965 966 967 968 969
      .def("set_place",
           [](platform::Place &self, const platform::CPUPlace &cpu_place) {
             self = cpu_place;
           })
      .def("set_place",
D
dzhwinter 已提交
970
           [](platform::Place &self, const platform::CUDAPlace &gpu_place) {
Y
Yu Yang 已提交
971
             self = gpu_place;
C
chengduoZH 已提交
972 973
           })
      .def("set_place", [](platform::Place &self,
C
chengduoZH 已提交
974 975
                           const platform::CUDAPinnedPlace &cuda_pinned_place) {
        self = cuda_pinned_place;
C
chengduoZH 已提交
976
      });
Y
Yu Yang 已提交
977

Y
Yu Yang 已提交
978 979 980
  py::class_<OperatorBase>(m, "Operator")
      .def_static("create",
                  [](py::bytes protobin) {
981
                    proto::OpDesc desc;
Y
Yu Yang 已提交
982 983 984 985 986
                    PADDLE_ENFORCE(desc.ParsePartialFromString(protobin),
                                   "Cannot parse user input to OpDesc");
                    PADDLE_ENFORCE(desc.IsInitialized(),
                                   "User OpDesc is not initialized, reason %s",
                                   desc.InitializationErrorString());
987
                    return OpRegistry::CreateOp(desc);
Y
Yu Yang 已提交
988
                  })
989
      .def("run",
990
           [](OperatorBase &self, const Scope &scope,
D
dzhwinter 已提交
991 992 993
              const platform::CPUPlace &place) { self.Run(scope, place); })
      .def("run",
           [](OperatorBase &self, const Scope &scope,
D
dzhwinter 已提交
994
              const platform::CUDAPlace &place) { self.Run(scope, place); })
C
chengduoZH 已提交
995 996 997 998 999
      .def("run",
           [](OperatorBase &self, const Scope &scope,
              const platform::CUDAPinnedPlace &place) {
             self.Run(scope, place);
           })
Y
Yu Yang 已提交
1000 1001 1002 1003 1004 1005 1006
      .def("type",
           [](const OperatorBase &op) -> std::string { return op.Type(); })
      .def("outputs",
           [](const OperatorBase &op)
               -> std::map<std::string, std::vector<std::string>> {
                 return op.Outputs();
               })
Q
qijun 已提交
1007 1008
      .def("output_vars",
           [](const OperatorBase &op) { return op.OutputVars(true); })
Y
Yu Yang 已提交
1009
      .def("inputs", [](const OperatorBase &op) { return op.Inputs(); })
Q
qijun 已提交
1010
      .def("input_vars", [](const OperatorBase &op) { return op.InputVars(); })
Y
Yu Yang 已提交
1011 1012 1013 1014
      .def("__str__", &OperatorBase::DebugString)
      .def("no_intermediate_outputs",
           [](const OperatorBase &op) { return op.OutputVars(false); })
      .def("support_gpu", &OperatorBase::SupportGPU);
Y
Yu Yang 已提交
1015

F
fengjiayi 已提交
1016
  py::class_<framework::Executor>(m, "Executor")
D
dzhwinter 已提交
1017
      .def(py::init<const platform::Place &>())
Y
Yancey1989 已提交
1018
      .def("close", &Executor::Close)
D
dongdaxiang 已提交
1019
      .def("run_from_dataset", &Executor::RunFromDataset)
S
sneaxiy 已提交
1020
      .def("run", [](Executor &self, const ProgramDesc &prog, Scope *scope,
S
sneaxiy 已提交
1021 1022
                     int block_id, bool create_local_scope, bool create_vars,
                     const std::vector<std::string> &fetch_vars) {
S
sneaxiy 已提交
1023
        pybind11::gil_scoped_release release;
S
sneaxiy 已提交
1024 1025
        self.Run(prog, scope, block_id, create_local_scope, create_vars,
                 fetch_vars);
S
sneaxiy 已提交
1026
      });
S
sneaxiy 已提交
1027

D
dzhwinter 已提交
1028
  m.def("init_gflags", framework::InitGflags);
Y
Yang Yu 已提交
1029
  m.def("init_glog", framework::InitGLOG);
1030
  m.def("init_dgc", framework::InitDGC);
X
Xin Pan 已提交
1031 1032
  m.def("init_devices",
        [](bool init_p2p) { framework::InitDevices(init_p2p); });
1033

1034
  m.def("is_compiled_with_ngraph", IsCompiledWithNGRAPH);
1035
  m.def("is_compiled_with_cuda", IsCompiledWithCUDA);
1036
  m.def("is_compiled_with_mkldnn", IsCompiledWithMKLDNN);
1037
  m.def("is_compiled_with_brpc", IsCompiledWithBrpc);
Y
update  
Yancey1989 已提交
1038
  m.def("is_compiled_with_dist", IsCompiledWithDIST);
1039 1040 1041 1042 1043 1044
#ifdef PADDLE_WITH_CUDA
  m.def("is_float16_supported", [](const platform::CUDAPlace &place) -> bool {
    // Only GPUs with Compute Capability >= 53 support float16
    return platform::GetCUDAComputeCapability(place.device) >= 53;
  });
#endif
1045

1046
  m.def("set_feed_variable", framework::SetFeedVariable);
Q
qijun 已提交
1047
  m.def("get_fetch_variable", framework::GetFetchVariable);
1048
  m.def("get_variable_tensor", framework::GetVariableTensor);
Q
qijun 已提交
1049

X
Xin Pan 已提交
1050 1051
  m.def("_is_program_version_supported", IsProgramVersionSupported);

1052 1053 1054 1055 1056
  BindProgramDesc(&m);
  BindBlockDesc(&m);
  BindVarDsec(&m);
  BindOpDesc(&m);
  BindConstValue(&m);
Y
Yu Yang 已提交
1057

Y
Yu Yang 已提交
1058 1059 1060 1061 1062 1063 1064 1065 1066
  py::class_<framework::LoDRankTable>(m, "LodRankTable")
      .def("items", [](framework::LoDRankTable &table) {
        std::vector<std::pair<size_t, size_t>> res;
        for (auto &item : table.items()) {
          res.push_back({item.index, item.length});
        }
        return res;
      });

Z
Zeng Jinle 已提交
1067 1068 1069 1070 1071 1072 1073 1074 1075 1076
  py::class_<LoDTensorArray>(m, "LoDTensorArray", R"DOC(
    Array of LoDTensor.

    Examples:
        .. code-block:: python
        
          import paddle.fluid as fluid

          arr = fluid.LoDTensorArray()
)DOC")
S
sneaxiy 已提交
1077 1078
      .def("__init__",
           [](LoDTensorArray &instance) { new (&instance) LoDTensorArray(); })
Y
Yu Yang 已提交
1079 1080 1081 1082 1083 1084 1085 1086 1087 1088
      .def("__getitem__",
           [](LoDTensorArray &self, size_t i) { return &self.at(i); },
           py::return_value_policy::reference)
      .def("__len__", [](LoDTensorArray &self) { return self.size(); })
      .def("__setitem__",
           [](LoDTensorArray &self, size_t i, const LoDTensor &t) {
             PADDLE_ENFORCE_LT(i, self.size());
             self[i].ShareDataWith(t);
             self[i].set_lod(t.lod());
           })
S
sneaxiy 已提交
1089 1090 1091 1092 1093 1094
      .def("append",
           [](LoDTensorArray &self, const LoDTensor &t) {
             self.emplace_back();
             self.back().ShareDataWith(t);
             self.back().set_lod(t.lod());
           },
Z
Zeng Jinle 已提交
1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108
           py::arg("tensor"), R"DOC(
             Append a LoDensor to LoDTensorArray.

             Examples:
                 .. code-block:: python

                   import paddle.fluid as fluid
                   import numpy as np

                   arr = fluid.LoDTensorArray()
                   t = fluid.LoDTensor()
                   t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                   arr.append(t)
           )DOC");
Y
Yu Yang 已提交
1109

D
dzhwinter 已提交
1110 1111 1112
  m.def("IsInplace",
        [](std::string op) -> bool { return operators::IsInplace(op); });

Y
Yu Yang 已提交
1113
  m.def("op_support_gpu", OpSupportGPU);
D
Dong Zhihong 已提交
1114
#ifdef PADDLE_WITH_CUDA
D
Dong Zhihong 已提交
1115
  m.def("get_cuda_device_count", platform::GetCUDADeviceCount);
D
dangqingqing 已提交
1116

P
peizhilin 已提交
1117
#ifndef _WIN32
D
dangqingqing 已提交
1118 1119 1120
  m.def("nvprof_init", platform::CudaProfilerInit);
  m.def("nvprof_start", platform::CudaProfilerStart);
  m.def("nvprof_stop", platform::CudaProfilerStop);
D
Dong Zhihong 已提交
1121
#endif
P
peizhilin 已提交
1122
#endif
Y
Yu Yang 已提交
1123

1124 1125 1126 1127
  py::enum_<platform::ProfilerState>(m, "ProfilerState", py::arithmetic())
      .value("kDisabled", platform::ProfilerState::kDisabled)
      .value("kCPU", platform::ProfilerState::kCPU)
      .value("kCUDA", platform::ProfilerState::kCUDA)
1128
      .value("kAll", platform::ProfilerState::kAll)
1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141
      .export_values();

  py::enum_<platform::EventSortingKey>(m, "EventSortingKey", py::arithmetic())
      .value("kDefault", platform::EventSortingKey::kDefault)
      .value("kCalls", platform::EventSortingKey::kCalls)
      .value("kTotal", platform::EventSortingKey::kTotal)
      .value("kMin", platform::EventSortingKey::kMin)
      .value("kMax", platform::EventSortingKey::kMax)
      .value("kAve", platform::EventSortingKey::kAve)
      .export_values();

  m.def("enable_profiler", platform::EnableProfiler);
  m.def("disable_profiler", platform::DisableProfiler);
X
Xin Pan 已提交
1142
  m.def("is_profiler_enabled", platform::IsProfileEnabled);
1143
  m.def("reset_profiler", platform::ResetProfiler);
1144
  m.def("get_pass", [](const std::string &pass_type) {
W
WangZhen 已提交
1145 1146 1147
    auto pass = framework::ir::PassRegistry::Instance().Get(pass_type);
    return std::shared_ptr<framework::ir::Pass>(std::move(pass));
  });
Y
Yu Yang 已提交
1148

1149 1150
  py::class_<ir::Pass, std::shared_ptr<ir::Pass>> pass(m, "Pass");
  pass.def(py::init())
W
WangZhen 已提交
1151
      .def("has", &ir::Pass::Has)
1152 1153 1154
      .def("set_not_owned",
           [](ir::Pass &self, const std::string &attr_name, ProgramDesc &attr) {
             self.SetNotOwned<ProgramDesc>(attr_name, &attr);
W
WangZhen 已提交
1155
           })
1156
      .def(
1157
          "set",
1158 1159 1160
          [](ir::Pass &self, const std::string &name, const std::string &attr) {
            self.Set<std::string>(name, new std::string(attr));
          })
1161 1162
      .def("set", [](ir::Pass &self, const std::string &name,
                     int val) { self.Set<const int>(name, new int(val)); })
F
flame 已提交
1163 1164
      .def("type", &ir::Pass::Type)
      .def("apply", [](ir::Pass &self, std::shared_ptr<ir::Graph> graph) {
1165
        self.Apply(graph.get());
F
flame 已提交
1166
      });
1167

X
fix  
Xin Pan 已提交
1168 1169
  py::class_<ir::PassBuilder, std::shared_ptr<ir::PassBuilder>> pb(
      m, "PassBuilder");
1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183
  pb.def(py::init())
      .def("append_pass",
           [](ir::PassBuilder &self,
              const std::string &pass_type) -> std::shared_ptr<ir::Pass> {
             return self.AppendPass(pass_type);
           })
      .def("all_passes", [](ir::PassBuilder &self) { return self.AllPasses(); })
      .def("insert_pass",
           [](ir::PassBuilder &self, size_t idx, const std::string &pass_type) {
             return self.InsertPass(idx, pass_type);
           })
      .def("remove_pass",
           [](ir::PassBuilder &self, size_t idx) { self.RemovePass(idx); });

Y
yuyang18 已提交
1184
  // -- python binds for parallel executor.
X
Xin Pan 已提交
1185

Y
yuyang18 已提交
1186
  py::class_<ParallelExecutor> pe(m, "ParallelExecutor");
C
chengduo 已提交
1187 1188 1189 1190
  py::class_<ExecutionStrategy> exec_strategy(pe, "ExecutionStrategy", R"DOC(
    ExecutionStrategy allows the user to more preciously control how to run
    the program in ParallelExecutor by setting the property.

C
chengduo 已提交
1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201
    Examples:
        .. code-block:: python

          exec_strategy = fluid.ExecutionStrategy()
          exec_strategy.num_threads = 4

          train_exe = fluid.ParallelExecutor(use_cuda=True,
                                             loss_name=loss.name,
                                             exec_strategy=exec_strategy)

          train_loss, = train_exe.run([loss.name], feed=feed_dict)
C
chengduo 已提交
1202 1203 1204

        )DOC");

Y
yuyang18 已提交
1205
  exec_strategy.def(py::init())
Y
yuyang18 已提交
1206 1207 1208 1209 1210
      .def_property(
          "num_threads",
          [](const ExecutionStrategy &self) { return self.num_threads_; },
          [](ExecutionStrategy &self, size_t num_threads) {
            self.num_threads_ = num_threads;
C
chengduo 已提交
1211 1212 1213 1214 1215 1216 1217 1218 1219 1220
          },
          R"DOC(The type is INT, num_threads represents the size of thread pool that
            used to run the operators of the current program in ParallelExecutor.
            If :math:`num\_threads=1`, all the operators will execute one by one,
            but the order maybe difference between iterations.
            If it is not set, it will be set in ParallelExecutor according to the
            device type and device count, for GPU, :math:`num\_threads=device\_count*4`, for CPU,
            :math:`num\_threads=CPU\_NUM*4`, the explanation of:math:`CPU\_NUM` is in ParallelExecutor.
            if it is not set, ParallelExecutor will get the cpu count by calling
            `multiprocessing.cpu_count()`. Default 0.)DOC")
Y
yuyang18 已提交
1221
      .def_property(
1222 1223 1224 1225
          "use_cuda",
          [](const ExecutionStrategy &self) { return self.use_cuda_; },
          [](ExecutionStrategy &self, bool use_cuda) {
            self.use_cuda_ = use_cuda;
C
chengduo 已提交
1226 1227 1228 1229
          })  // FIXME(chengduo): Doesn't add doc for 'use_cuda', use_cuda may
      // make user confuse, because ParallelExecutor has a parameter named
      // 'use_cuda' too, in current implementation, ParallelExecutor's
      // 'use_cuda' will rewrite ExecutionStrategy's 'use_cuda'.
Y
yuyang18 已提交
1230 1231 1232 1233 1234
      .def_property(
          "allow_op_delay",
          [](const ExecutionStrategy &self) { return self.allow_op_delay_; },
          [](ExecutionStrategy &self, bool allow_op_delay) {
            self.allow_op_delay_ = allow_op_delay;
C
chengduo 已提交
1235 1236 1237 1238
          },
          R"DOC(The type is BOOL, allow_op_delay represents whether to delay the
                communication operators to run, it may make the execution faster.
                Note that in some models, allow_op_delay may cause program hang. Default False.)DOC")
Y
yuyang18 已提交
1239 1240 1241 1242 1243 1244 1245
      .def_property(
          "num_iteration_per_drop_scope",
          [](const ExecutionStrategy &self) {
            return self.num_iteration_per_drop_scope_;
          },
          [](ExecutionStrategy &self, size_t num_iteration_per_drop_scope) {
            self.num_iteration_per_drop_scope_ = num_iteration_per_drop_scope;
C
chengduo 已提交
1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256
          },
          R"DOC(The type is INT, num_iteration_per_drop_scope indicates how
                many iterations to clean up the temp variables which
                is generated during execution. It may make the execution faster,
                because the temp variable's shape maybe the same between two iterations. Default 100.

                NOTES:
                    1. If you fetch data when calling the 'run', the ParallelExecutor
                       will clean up the temp variables at the end of the current iteration.
                    2. In some NLP model, it may cause the GPU memory is insufficient,
                       in this case, you should reduce `num_iteration_per_drop_scope`.
1257
              )DOC")
Q
Qiao Longfei 已提交
1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268
      .def_property(
          "num_iteration_per_run",
          [](const ExecutionStrategy &self) {
            return self.num_iteration_per_run_;
          },
          [](ExecutionStrategy &self, size_t num_iteration_per_run) {
            self.num_iteration_per_run_ = num_iteration_per_run;
          },
          R"DOC(This config that how many iteration the executor will run when
                user call pe.run() in python
              )DOC")
1269 1270 1271 1272 1273
      .def_property("_dry_run",
                    [](const ExecutionStrategy &self) { return self.dry_run_; },
                    [](ExecutionStrategy &self, bool dry_run) {
                      self.dry_run_ = dry_run;
                    });
C
chengduo 已提交
1274

Y
yuyang18 已提交
1275
  exec_strategy.def_property(
Y
yuyang18 已提交
1276 1277 1278 1279 1280 1281 1282
      "use_experimental_executor",
      [](const ExecutionStrategy &self) {
        return self.type_ == ExecutionStrategy::kExperimental;
      },
      [](ExecutionStrategy &self, bool experimental) {
        self.type_ = experimental ? ExecutionStrategy::kExperimental
                                  : ExecutionStrategy::kDefault;
Y
yuyang18 已提交
1283 1284
      });

C
chengduo 已提交
1285 1286 1287 1288
  py::class_<BuildStrategy> build_strategy(pe, "BuildStrategy", R"DOC(
    BuildStrategy allows the user to more preciously control how to
    build the SSA Graph in ParallelExecutor by setting the property.

C
chengduo 已提交
1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299
    Examples:
        .. code-block:: python

          build_strategy = fluid.BuildStrategy()
          build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce

          train_exe = fluid.ParallelExecutor(use_cuda=True,
                                             loss_name=loss.name,
                                             build_strategy=build_strategy)

          train_loss, = train_exe.run([loss.name], feed=feed_dict)
C
chengduo 已提交
1300
)DOC");
Y
yuyang18 已提交
1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316

  py::enum_<BuildStrategy::ReduceStrategy>(build_strategy, "ReduceStrategy")
      .value("Reduce", BuildStrategy::ReduceStrategy::kReduce)
      .value("AllReduce", BuildStrategy::ReduceStrategy::kAllReduce);
  py::enum_<BuildStrategy::GradientScaleStrategy>(build_strategy,
                                                  "GradientScaleStrategy")
      .value("CoeffNumDevice",
             BuildStrategy::GradientScaleStrategy::kCoeffNumDevice)
      .value("One", BuildStrategy::GradientScaleStrategy::kOne)
      .value("Customized", BuildStrategy::GradientScaleStrategy::kCustomized);

  build_strategy.def(py::init())
      .def_property(
          "reduce_strategy",
          [](const BuildStrategy &self) { return self.reduce_; },
          [](BuildStrategy &self, BuildStrategy::ReduceStrategy strategy) {
X
Xin Pan 已提交
1317
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
Y
yuyang18 已提交
1318
            self.reduce_ = strategy;
C
chengduo 已提交
1319 1320 1321 1322 1323 1324 1325
          },
          R"DOC(The type is STR, there are two reduce strategies in ParallelExecutor,
                  'AllReduce' and 'Reduce'. If you want that all the parameters'
                  optimization are done on all devices independently, you should choose 'AllReduce';
                  if you choose 'Reduce', all the parameters' optimization will be evenly distributed
                  to different devices, and then broadcast the optimized parameter to other devices.
                  In some models, `Reduce` is faster. Default 'AllReduce'. )DOC")
Y
yuyang18 已提交
1326 1327 1328 1329 1330
      .def_property(
          "gradient_scale_strategy",
          [](const BuildStrategy &self) { return self.gradient_scale_; },
          [](BuildStrategy &self,
             BuildStrategy::GradientScaleStrategy strategy) {
X
Xin Pan 已提交
1331
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
Y
yuyang18 已提交
1332
            self.gradient_scale_ = strategy;
C
chengduo 已提交
1333 1334 1335 1336 1337 1338
          },
          R"DOC(The type is STR, there are three ways of defining :math:`loss@grad` in
                   ParallelExecutor, 'CoeffNumDevice', 'One' and 'Customized'. By default,
                   ParallelExecutor sets the :math:`loss@grad` according to the number of devices.
                   If you want to customize :math:`loss@grad`, you can choose 'Customized'.
                   Default 'CoeffNumDevice'.)DOC")
Y
yuyang18 已提交
1339 1340 1341 1342
      .def_property(
          "debug_graphviz_path",
          [](const BuildStrategy &self) { return self.debug_graphviz_path_; },
          [](BuildStrategy &self, const std::string &path) {
X
Xin Pan 已提交
1343
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
Y
yuyang18 已提交
1344
            self.debug_graphviz_path_ = path;
C
chengduo 已提交
1345 1346 1347 1348
          },
          R"DOC(The type is STR, debug_graphviz_path indicate the path that
                    writing the SSA Graph to file in the form of graphviz, you.
                    It is useful for debugging. Default "")DOC")
S
sneaxiy 已提交
1349 1350 1351 1352 1353 1354
      .def_property(
          "enable_sequential_execution",
          [](const BuildStrategy &self) {
            return self.enable_sequential_execution_;
          },
          [](BuildStrategy &self, bool b) {
X
Xin Pan 已提交
1355
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
S
sneaxiy 已提交
1356 1357 1358 1359 1360 1361 1362 1363 1364
            self.enable_sequential_execution_ = b;
          },
          R"DOC(The type is BOOL. If set True, the execution order of ops would be the same as what is in the program. Default False.)DOC")
      .def_property(
          "remove_unnecessary_lock",
          [](const BuildStrategy &self) {
            return self.remove_unnecessary_lock_;
          },
          [](BuildStrategy &self, bool b) {
X
Xin Pan 已提交
1365
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
S
sneaxiy 已提交
1366 1367
            self.remove_unnecessary_lock_ = b;
          },
S
sneaxiy 已提交
1368
          R"DOC(The type is BOOL. If set True, some locks in GPU ops would be released and ParallelExecutor would run faster. Default True.)DOC")
1369 1370 1371 1372 1373 1374
      .def_property(
          "num_trainers",
          [](const BuildStrategy &self) { return self.num_trainers_; },
          [](BuildStrategy &self, int num_trainers) {
            self.num_trainers_ = num_trainers;
          })
1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386
      .def_property(
          "trainers_endpoints",
          [](const BuildStrategy &self) { return self.trainers_endpoints_; },
          [](BuildStrategy &self,
             const std::vector<std::string> &trainers_endpoints) {
            self.trainers_endpoints_ = trainers_endpoints;
          })
      .def_property("trainer_id",
                    [](const BuildStrategy &self) { return self.trainer_id_; },
                    [](BuildStrategy &self, int trainer_id) {
                      self.trainer_id_ = trainer_id;
                    })
C
chengduo 已提交
1387 1388 1389 1390 1391 1392
      .def_property(
          "fuse_elewise_add_act_ops",
          [](const BuildStrategy &self) {
            return self.fuse_elewise_add_act_ops_;
          },
          [](BuildStrategy &self, bool b) {
X
Xin Pan 已提交
1393
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
C
chengduo 已提交
1394 1395 1396 1397 1398
            self.fuse_elewise_add_act_ops_ = b;
          },
          R"DOC(The type is BOOL, fuse_elewise_add_act_ops indicate whether
                     to fuse elementwise_add_op and activation_op,
                     it may make the execution faster. Default False)DOC")
1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411
      .def_property(
          "fuse_relu_depthwise_conv",
          [](const BuildStrategy &self) {
            return self.fuse_relu_depthwise_conv_;
          },
          [](BuildStrategy &self, bool b) {
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
            self.fuse_relu_depthwise_conv_ = b;
          },
          R"DOC(The type is BOOL, fuse_relu_depthwise_conv indicate whether
                      to fuse relu and depthwise_conv2d,
                      it will save GPU memory and may make the execution faster.
                      This options is only available in GPU devices.
1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425
                      Default False.)DOC")
      .def_property(
          "fuse_broadcast_ops",
          [](const BuildStrategy &self) { return self.fuse_broadcast_ops_; },
          [](BuildStrategy &self, bool b) {
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
            self.fuse_broadcast_ops_ = b;
          },
          R"DOC(The type is BOOL, fuse_broadcast_op indicates whether
                      to fuse the broadcast ops. Note that, in Reduce mode,
                      fusing broadcast ops may make the program faster. Because
                      fusing broadcast OP equals delaying the execution of all
                      broadcast Ops, in this case, all nccl streams are used only
                      for NCCLReduce operations for a period of time. Default False.)DOC")
C
chengduo 已提交
1426 1427 1428 1429 1430 1431 1432 1433 1434
      .def_property("fuse_all_optimizer_ops",
                    [](const BuildStrategy &self) {
                      return self.fuse_all_optimizer_ops_;
                    },
                    [](BuildStrategy &self, bool b) {
                      PADDLE_ENFORCE(!self.IsFinalized(),
                                     "BuildStrategy is finlaized.");
                      self.fuse_all_optimizer_ops_ = b;
                    })
Q
qingqing01 已提交
1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449
      .def_property(
          "sync_batch_norm",
          [](const BuildStrategy &self) { return self.sync_batch_norm_; },
          [](BuildStrategy &self, bool b) {
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
            self.sync_batch_norm_ = b;
          },
          R"DOC(The type is BOOL, sync_batch_norm indicates whether to use
                synchronous batch normalization which synchronizes the mean
                and variance through multi-devices in training phase.

                Current implementation doesn't support FP16 training and CPU.
                And only synchronous on one machine, not all machines.

                Default False)DOC")
D
dzhwinter 已提交
1450 1451 1452
      .def_property(
          "memory_optimize",
          [](const BuildStrategy &self) { return self.memory_optimize_; },
1453 1454 1455 1456 1457 1458 1459 1460 1461 1462
          [](BuildStrategy &self, bool b) { self.memory_optimize_ = b; },
          R"DOC(The type is BOOL, memory opitimize aims to save total memory 
                consumption, set to True to enable it.
                
                Memory Optimize is our experimental feature, some variables 
                may be reused/removed by optimize strategy. If you need to
                fetch some variable values when using this feature, please
                set the persistable property of the variables to True.
                
                Default False)DOC")
1463 1464 1465 1466
      .def_property(
          "is_distribution",
          [](const BuildStrategy &self) { return self.is_distribution_; },
          [](BuildStrategy &self, bool b) { self.is_distribution_ = b; })
Q
can run  
Qiao Longfei 已提交
1467 1468 1469
      .def_property("async_mode",
                    [](const BuildStrategy &self) { return self.async_mode_; },
                    [](BuildStrategy &self, bool b) { self.async_mode_ = b; })
D
dzhwinter 已提交
1470
      .def_property(
D
dzhwinter 已提交
1471 1472 1473
          "enable_inplace",
          [](const BuildStrategy &self) { return self.enable_inplace_; },
          [](BuildStrategy &self, bool b) { self.enable_inplace_ = b; })
C
chengduo 已提交
1474 1475 1476 1477
      .def_property(
          "fuse_all_reduce_ops",
          [](const BuildStrategy &self) { return self.fuse_all_reduce_ops_; },
          [](BuildStrategy &self, bool b) { self.fuse_all_reduce_ops_ = b; })
1478 1479 1480 1481
      .def_property(
          "cache_runtime_context",
          [](const BuildStrategy &self) { return self.cache_runtime_context_; },
          [](BuildStrategy &self, bool b) { self.cache_runtime_context_ = b; })
1482 1483 1484 1485
      .def_property(
          "cache_expected_kernel",
          [](const BuildStrategy &self) { return self.cache_expected_kernel_; },
          [](BuildStrategy &self, bool b) { self.cache_expected_kernel_ = b; })
1486
      .def("_finalize_strategy_and_create_passes",
X
fix  
Xin Pan 已提交
1487
           [](BuildStrategy &self) -> std::shared_ptr<ir::PassBuilder> {
1488 1489 1490 1491 1492
             return self.CreatePassesFromStrategy(true);
           },
           R"DOC(Allow user to customized passes. Normally model-specific
                optimization passes should be defined in this way. BuildStrategy
                cannot be updated after being finalized.)DOC");
Y
yuyang18 已提交
1493 1494

  pe.def(py::init<const std::vector<platform::Place> &,
Y
Yan Xu 已提交
1495
                  const std::vector<std::string> &, const std::string &,
X
Xin Pan 已提交
1496
                  Scope *, std::vector<Scope *> &, const ExecutionStrategy &,
X
Xin Pan 已提交
1497
                  const BuildStrategy &, ir::Graph *>())
Y
Yu Yang 已提交
1498 1499 1500 1501
      // NOTE: even we return a vec<Scope*>* to Python use reference policy.
      // We still cannot get local_scope from this vector, since the element
      // of vec<Scope*> will be freed by Python GC. We can only return Scope*
      // one by one and mark them as reference.
1502 1503 1504 1505 1506
      .def("local_scopes",
           [](ParallelExecutor &self) -> std::vector<Scope *> * {
             return &self.GetLocalScopes();
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
1507 1508 1509 1510
      .def("feed_tensors_into_local_scopes",
           &ParallelExecutor::FeedTensorsIntoLocalScopes)
      .def("feed_and_split_tensor_into_local_scopes",
           &ParallelExecutor::FeedAndSplitTensorIntoLocalScopes)
S
sneaxiy 已提交
1511 1512 1513 1514 1515 1516
      .def("run", [](ParallelExecutor &self,
                     const std::vector<std::string> &fetch_tensors,
                     const std::string &fetched_var_name) {
        pybind11::gil_scoped_release release;
        self.Run(fetch_tensors, fetched_var_name);
      });
Y
Yu Yang 已提交
1517

1518
  BindRecordIOWriter(&m);
W
Wang Guibao 已提交
1519
  BindAsyncExecutor(&m);
D
dongdaxiang 已提交
1520
  BindFleetWrapper(&m);
W
wopeizl 已提交
1521
#ifndef _WIN32
D
dongdaxiang 已提交
1522
  BindNCCLWrapper(&m);
W
wopeizl 已提交
1523
#endif
F
flame 已提交
1524 1525
  BindGraph(&m);
  BindNode(&m);
F
flame 已提交
1526
  BindInferenceApi(&m);
1527
  BindDataset(&m);
L
Luo Tao 已提交
1528
}
1529
}  // namespace pybind
1530
}  // namespace paddle