pybind.cc 67.0 KB
Newer Older
1
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
2 3 4 5 6

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

7
http://www.apache.org/licenses/LICENSE-2.0
8 9 10 11 12 13

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
L
lgone2000 已提交
14
#include <Python.h>
C
chengduoZH 已提交
15 16
#include <algorithm>
#include <map>
S
sneaxiy 已提交
17
#include <memory>
C
chengduoZH 已提交
18 19 20
#include <mutex>  // NOLINT // for call_once
#include <string>
#include <unordered_map>
21
#include <unordered_set>
C
chengduoZH 已提交
22 23
#include <utility>
#include <vector>
24

Y
Yi Wang 已提交
25 26 27
#include "paddle/fluid/framework/executor.h"
#include "paddle/fluid/framework/feed_fetch_method.h"
#include "paddle/fluid/framework/framework.pb.h"
S
sneaxiy 已提交
28
#include "paddle/fluid/framework/garbage_collector.h"
29
#include "paddle/fluid/framework/ir/alloc_continuous_space_for_grad_pass.h"
30
#include "paddle/fluid/framework/ir/pass_builder.h"
Y
Yi Wang 已提交
31 32 33
#include "paddle/fluid/framework/lod_rank_table.h"
#include "paddle/fluid/framework/lod_tensor.h"
#include "paddle/fluid/framework/lod_tensor_array.h"
S
sneaxiy 已提交
34
#include "paddle/fluid/framework/op_info.h"
35
#include "paddle/fluid/framework/op_registry.h"
Y
Yu Yang 已提交
36
#include "paddle/fluid/framework/parallel_executor.h"
Y
Yi Wang 已提交
37
#include "paddle/fluid/framework/prune.h"
Y
Refine  
Yu Yang 已提交
38
#include "paddle/fluid/framework/reader.h"
S
sneaxiy 已提交
39
#include "paddle/fluid/framework/scope_pool.h"
Y
Yi Wang 已提交
40
#include "paddle/fluid/framework/selected_rows.h"
X
Xin Pan 已提交
41
#include "paddle/fluid/framework/version.h"
42
#include "paddle/fluid/imperative/layer.h"
M
minqiyang 已提交
43
#include "paddle/fluid/imperative/profiler.h"
Y
Refine  
Yu Yang 已提交
44
#include "paddle/fluid/memory/allocation/allocator_strategy.h"
45
#include "paddle/fluid/memory/allocation/legacy_allocator.h"
D
dzhwinter 已提交
46
#include "paddle/fluid/operators/activation_op.h"
S
sneaxiy 已提交
47
#include "paddle/fluid/operators/py_func_op.h"
S
sneaxiy 已提交
48
#include "paddle/fluid/operators/reader/lod_tensor_blocking_queue.h"
Y
Yu Yang 已提交
49
#include "paddle/fluid/platform/cpu_info.h"
Y
Yi Wang 已提交
50
#include "paddle/fluid/platform/enforce.h"
51
#include "paddle/fluid/platform/init.h"
Y
Yi Wang 已提交
52 53
#include "paddle/fluid/platform/place.h"
#include "paddle/fluid/platform/profiler.h"
W
Wang Guibao 已提交
54
#include "paddle/fluid/pybind/async_executor_py.h"
Y
Yi Wang 已提交
55
#include "paddle/fluid/pybind/const_value.h"
D
dongdaxiang 已提交
56
#include "paddle/fluid/pybind/data_set_py.h"
Y
Yi Wang 已提交
57
#include "paddle/fluid/pybind/exception.h"
D
dongdaxiang 已提交
58
#include "paddle/fluid/pybind/fleet_wrapper_py.h"
59
#include "paddle/fluid/pybind/imperative.h"
F
flame 已提交
60
#include "paddle/fluid/pybind/inference_api.h"
F
flame 已提交
61
#include "paddle/fluid/pybind/ir.h"
W
wopeizl 已提交
62
#ifndef _WIN32
D
dongdaxiang 已提交
63
#include "paddle/fluid/pybind/nccl_wrapper_py.h"
W
wopeizl 已提交
64
#endif
65 66
#include "paddle/fluid/pybind/protobuf.h"
#include "paddle/fluid/pybind/pybind.h"  // NOLINT
S
sneaxiy 已提交
67
#include "paddle/fluid/pybind/reader_py.h"
Y
Yu Yang 已提交
68
#include "paddle/fluid/pybind/recordio.h"
Y
Yi Wang 已提交
69
#include "paddle/fluid/pybind/tensor_py.h"
70
#include "paddle/fluid/string/to_string.h"
71

D
Dong Zhihong 已提交
72
#ifdef PADDLE_WITH_CUDA
P
peizhilin 已提交
73
#ifndef _WIN32
Y
Yi Wang 已提交
74
#include "paddle/fluid/operators/nccl/nccl_gpu_common.h"
P
peizhilin 已提交
75
#endif
Y
Yi Wang 已提交
76 77
#include "paddle/fluid/platform/cuda_profiler.h"
#include "paddle/fluid/platform/gpu_info.h"
D
Dong Zhihong 已提交
78 79
#endif

80 81 82 83
#ifdef PADDLE_WITH_DISTRIBUTE
#include "paddle/fluid/pybind/communicator_py.h"
#endif

M
minqiyang 已提交
84 85
#include "pybind11/stl.h"

86 87 88 89
DEFINE_bool(reader_queue_speed_test_mode, false,
            "If set true, the queue.pop will only get data from queue but not "
            "remove the data from queue for speed testing");

Q
Qiao Longfei 已提交
90 91 92
// disable auto conversion to list in Python
PYBIND11_MAKE_OPAQUE(paddle::framework::LoDTensorArray);

93
namespace paddle {
94
namespace pybind {
95
bool IsCompiledWithCUDA() {
96
#ifndef PADDLE_WITH_CUDA
Q
qijun 已提交
97 98 99 100 101 102
  return false;
#else
  return true;
#endif
}

103 104 105 106 107 108 109 110
bool IsCompiledWithMKLDNN() {
#ifndef PADDLE_WITH_MKLDNN
  return false;
#else
  return true;
#endif
}

111 112 113 114 115 116 117 118
bool IsCompiledWithNGRAPH() {
#ifndef PADDLE_WITH_NGRAPH
  return false;
#else
  return true;
#endif
}

119
bool IsCompiledWithBrpc() {
120
#ifndef PADDLE_WITH_DISTRIBUTE
121 122
  return false;
#endif
123 124 125 126 127 128

#ifdef PADDLE_WITH_GRPC
  return false;
#endif

  return true;
129 130
}

Y
update  
Yancey1989 已提交
131
bool IsCompiledWithDIST() {
Y
Yancey1989 已提交
132
#ifdef PADDLE_WITH_DISTRIBUTE
Y
update  
Yancey1989 已提交
133 134 135 136 137 138
  return true;
#else
  return false;
#endif
}

S
sneaxiy 已提交
139 140 141 142 143 144 145 146 147 148
template <typename PlaceType1, typename PlaceType2>
static inline bool IsSamePlace(const PlaceType1 &p1, const PlaceType2 &p2) {
  return paddle::platform::Place(p1) == paddle::platform::Place(p2);
}

template <typename PlaceType>
static inline int PlaceIndex(const PlaceType &p) {
  return static_cast<int>(paddle::platform::Place(p).which());
}

149
PYBIND11_MODULE(core, m) {
Y
Yu Yang 已提交
150 151 152
  // Not used, just make sure cpu_info.cc is linked.
  paddle::platform::CpuTotalPhysicalMemory();

Y
Refine  
Yu Yang 已提交
153
  paddle::memory::allocation::UseAllocatorStrategyGFlag();
S
sneaxiy 已提交
154

155
  m.doc() = "C++ core of PaddlePaddle";
156

157 158 159 160
  // using framework in this function. Since it is inside a function, it will
  // not cause namespace pollution.
  using namespace paddle::framework;  // NOLINT

161
  BindException(&m);
Y
Yu Yang 已提交
162

S
sneaxiy 已提交
163
  m.def(
S
sneaxiy 已提交
164
      "_append_python_callable_object_and_return_id",
S
sneaxiy 已提交
165 166 167 168
      [](py::object py_obj) -> size_t {
        return paddle::operators::AppendPythonCallableObjectAndReturnId(py_obj);
      });

S
sneaxiy 已提交
169 170 171
  m.def("_get_use_default_grad_op_desc_maker_ops",
        [] { return OpInfoMap::Instance().GetUseDefaultGradOpDescMakerOps(); });

S
sneaxiy 已提交
172 173 174
  // NOTE(zjl): ctest would load environment variables at the beginning even
  // though we have not `import paddle.fluid as fluid`. So we add this API
  // to enable eager deletion mode in unittest.
S
sneaxiy 已提交
175
  m.def("_set_eager_deletion_mode", &paddle::framework::SetEagerDeletionMode);
S
sneaxiy 已提交
176

177
  m.def("_set_fuse_parameter_group_size",
178
        &paddle::framework::ir::SetFuseParameterGroupsSize);
179
  m.def("_set_fuse_parameter_memory_size",
180
        &paddle::framework::ir::SetFuseParameterMemorySize);
181

S
sneaxiy 已提交
182 183 184
  m.add_object("_cleanup",
               py::capsule([]() { ScopePool::Instance().Clear(); }));

185 186 187 188 189 190 191
  m.def("get_mem_usage", [](int device) {
    return memory::allocation::GPUMemMonitor.GetMemUsage(device);
  });

  m.def("print_mem_usage",
        []() { return memory::allocation::GPUMemMonitor.PrintMemUsage(); });

192 193 194 195 196 197 198 199 200 201 202 203
  py::class_<imperative::detail::BackwardStrategy> backward_strategy(
      m, "BackwardStrategy", R"DOC()DOC");
  backward_strategy.def(py::init())
      .def_property("sort_sum_gradient",
                    [](const imperative::detail::BackwardStrategy &self) {
                      return self.sorted_sum_gradient_;
                    },
                    [](imperative::detail::BackwardStrategy &self,
                       bool sorted_sum_gradient) {
                      self.sorted_sum_gradient_ = sorted_sum_gradient;
                    });

M
minqiyang 已提交
204
  m.def("start_imperative_gperf_profiler",
M
minqiyang 已提交
205 206
        []() { imperative::StartProfile(); });

M
minqiyang 已提交
207
  m.def("stop_imperative_gperf_profiler", []() { imperative::StopProfile(); });
M
minqiyang 已提交
208

M
minqiyang 已提交
209
  py::class_<imperative::VarBase>(m, "VarBase", R"DOC()DOC")
210 211 212 213 214 215 216 217
      .def(
          py::init<const std::string &, paddle::framework::proto::VarType::Type,
                   const std::vector<int64_t>, const paddle::platform::CPUPlace,
                   bool, bool>())
      .def(
          py::init<const std::string &, paddle::framework::proto::VarType::Type,
                   const std::vector<int64_t>,
                   const paddle::platform::CUDAPlace, bool, bool>())
218
      .def("_run_backward",
219 220 221 222
           [](imperative::VarBase &self,
              const imperative::detail::BackwardStrategy &bckst) {
             self.RunBackward(bckst);
           })
M
minqiyang 已提交
223
      .def("_grad_name", &imperative::VarBase::GradName)
M
minqiyang 已提交
224
      .def("_grad_value", &imperative::VarBase::GradValue)
X
Xin Pan 已提交
225
      .def("_clear_gradient", &imperative::VarBase::ClearGradient)
M
minqiyang 已提交
226
      .def("_grad_ivar",
M
minqiyang 已提交
227
           [](const imperative::VarBase &self) { return self.grads_; },
M
minqiyang 已提交
228
           py::return_value_policy::reference)
M
minqiyang 已提交
229
      .def("_copy_to",
P
Paddle CI 已提交
230
           [](const imperative::VarBase &self, const platform::CPUPlace &place,
M
minqiyang 已提交
231 232 233 234 235
              bool blocking) {
             std::unique_ptr<imperative::VarBase> new_var =
                 self.NewVarBase(place, blocking);
             return new_var.release();
           },
P
Paddle CI 已提交
236
           py::return_value_policy::take_ownership)
M
minqiyang 已提交
237
      .def("_copy_to",
P
Paddle CI 已提交
238
           [](const imperative::VarBase &self, const platform::CUDAPlace &place,
M
minqiyang 已提交
239 240 241 242 243
              bool blocking) {
             std::unique_ptr<imperative::VarBase> new_var =
                 self.NewVarBase(place, blocking);
             return new_var.release();
           },
M
minqiyang 已提交
244
           py::return_value_policy::take_ownership)
245 246
      .def("value",
           [](const imperative::VarBase &self) { return self.var_.get(); },
M
minqiyang 已提交
247
           py::return_value_policy::reference)
248 249 250
      .def_property("name", &imperative::VarBase::Name,
                    &imperative::VarBase::SetName)
      .def_property_readonly("shape", &imperative::VarBase::Shape)
M
minqiyang 已提交
251
      .def_property_readonly("dtype", &imperative::VarBase::DataType)
252 253 254 255
      .def_property("persistable", &imperative::VarBase::IsPersistable,
                    &imperative::VarBase::SetPersistable)
      .def_property("stop_gradient", &imperative::VarBase::IsStopGradient,
                    &imperative::VarBase::SetStopGradient);
256

257
  py::class_<imperative::OpBase, PyOpBase>(m, "OpBase", R"DOC()DOC")
258
      .def(py::init<const std::string &>())
259
      .def("register_backward_hooks",
Y
Yan Xu 已提交
260 261 262
           [](imperative::OpBase &self, const py::object &callable) {
             self.RegisterBackwardHooks(callable);
           })
M
minqiyang 已提交
263 264 265 266 267 268 269 270 271 272
      .def_property("_trace_id",
                    [](const imperative::OpBase &self) {
                      pybind11::gil_scoped_release release;
                      return self.trace_id_;
                    },
                    [](imperative::OpBase &self, int trace_id) {
                      pybind11::gil_scoped_release release;
                      self.trace_id_ = trace_id;
                    },
                    py::return_value_policy::reference)
X
Xin Pan 已提交
273 274 275 276 277 278
      .def_property(
          "forward_id",
          [](const imperative::OpBase &self) { return self.forward_id_; },
          [](imperative::OpBase &self, int forward_id) {
            self.forward_id_ = forward_id;
          },
X
Xin Pan 已提交
279
          py::return_value_policy::reference)
X
polish  
Xin Pan 已提交
280
      .def_property_readonly("type", &imperative::OpBase::Type)
X
Xin Pan 已提交
281 282 283 284 285 286
      .def_property(
          "backward_id",
          [](const imperative::OpBase &self) { return self.backward_id_; },
          [](imperative::OpBase &self, int backward_id) {
            self.backward_id_ = backward_id;
          },
287 288
          py::return_value_policy::reference);

X
Xin Pan 已提交
289
  py::class_<imperative::Layer, Layer /* <--- trampoline*/> layer(m, "Layer");
290
  layer.def(py::init<>())
X
Xin Pan 已提交
291
      .def("forward", [](imperative::Layer &self,
292
                         const std::vector<imperative::VarBase *> &inputs) {
X
Xin Pan 已提交
293
        return self.Forward(inputs);
X
Xin Pan 已提交
294
      });
X
Xin Pan 已提交
295

X
polish  
Xin Pan 已提交
296
  py::class_<imperative::PyLayer>(m, "PyLayer")
X
Xin Pan 已提交
297
      .def(py::init<>())
X
Xin Pan 已提交
298 299
      .def_static(
          "apply",
X
Xin Pan 已提交
300
          [](int func_id, const std::vector<imperative::VarBase *> &inputs)
X
Xin Pan 已提交
301
              -> std::vector<imperative::VarBase *> {
302 303 304 305 306
                auto ret_vars = imperative::PyLayer::Apply(func_id, inputs);
                std::vector<imperative::VarBase *> outputs;
                outputs.reserve(ret_vars.size());
                for (size_t i = 0U; i != ret_vars.size(); ++i) {
                  // TODO(minqiyang): use unique_name generator to set a name
307 308
                  outputs.emplace_back(new imperative::VarBase(
                      "", std::move(ret_vars[i]), nullptr, true));
309 310 311
                }

                return outputs;
X
Xin Pan 已提交
312 313
              },
          py::return_value_policy::take_ownership)
X
polish  
Xin Pan 已提交
314 315 316 317 318
      .def_static("register_func",
                  [](int func_id, const py::object &callable) {
                    imperative::PyLayer::RegisterFunc(func_id, callable);
                  })
      .def_static("num_funcs", &imperative::PyLayer::NumFuncs);
X
Xin Pan 已提交
319

320
  BindImperative(&m);
321

322
  py::class_<Tensor>(m, "Tensor", py::buffer_protocol())
323
      .def("__array__", [](Tensor &self) { return TensorToPyArray(self); })
S
sneaxiy 已提交
324 325
      .def("_is_initialized",
           [](const Tensor &self) { return self.IsInitialized(); })
Y
yuyang18 已提交
326
      .def("_get_dims",
327
           [](const Tensor &self) { return vectorize(self.dims()); })
Y
yuyang18 已提交
328
      .def("_set_dims",
Q
qijun 已提交
329
           [](Tensor &self, const std::vector<int64_t> &dim) {
Y
Yu Yang 已提交
330
             self.Resize(make_ddim(dim));
Y
Yu Yang 已提交
331
           })
Y
yuyang18 已提交
332
      .def("_set_layout",
D
dzhwinter 已提交
333 334 335
           [](Tensor &self, const std::string &layout) {
             self.set_layout(StringToDataLayout(layout));
           })
Y
yuyang18 已提交
336
      .def("_alloc_float",
D
dzhwinter 已提交
337
           [](Tensor &self, paddle::platform::CUDAPlace &place) {
Q
qijun 已提交
338
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
339
           })
Y
yuyang18 已提交
340
      .def("_alloc_float",
Y
Yu Yang 已提交
341
           [](Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
342
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
343
           })
Y
yuyang18 已提交
344
      .def("_alloc_int",
Y
Yu Yang 已提交
345
           [](Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
346
             self.mutable_data<int>(place);
Y
Yu Yang 已提交
347
           })
Y
yuyang18 已提交
348
      .def("_alloc_int",
D
dzhwinter 已提交
349
           [](Tensor &self, paddle::platform::CUDAPlace &place) {
Q
qijun 已提交
350
             self.mutable_data<int>(place);
Q
qijun 已提交
351
           })
Y
yuyang18 已提交
352
      .def("_alloc_int",
C
chengduoZH 已提交
353 354 355
           [](Tensor &self, paddle::platform::CUDAPinnedPlace &place) {
             self.mutable_data<int>(place);
           })
Y
yuyang18 已提交
356
      .def("_alloc_float",
C
chengduoZH 已提交
357 358 359
           [](Tensor &self, paddle::platform::CUDAPinnedPlace &place) {
             self.mutable_data<float>(place);
           })
Z
Zeng Jinle 已提交
360
      .def("_clear", &Tensor::clear)
Y
Yu Yang 已提交
361 362
      .def("set", PyCPUTensorSetFromArray<float>)
      .def("set", PyCPUTensorSetFromArray<int>)
363
      .def("set", PyCPUTensorSetFromArray<double>)
364
      .def("set", PyCPUTensorSetFromArray<int64_t>)
Y
Yu Yang 已提交
365
      .def("set", PyCPUTensorSetFromArray<bool>)
366
      .def("set", PyCPUTensorSetFromArray<uint16_t>)
F
fengjiayi 已提交
367
      .def("set", PyCPUTensorSetFromArray<uint8_t>)
Q
qingqing01 已提交
368
      .def("set", PyCPUTensorSetFromArray<int8_t>)
369
#ifdef PADDLE_WITH_CUDA
Y
Yu Yang 已提交
370 371
      .def("set", PyCUDATensorSetFromArray<float>)
      .def("set", PyCUDATensorSetFromArray<int>)
372
      .def("set", PyCUDATensorSetFromArray<double>)
373
      .def("set", PyCUDATensorSetFromArray<int64_t>)
Y
Yu Yang 已提交
374
      .def("set", PyCUDATensorSetFromArray<bool>)
375
      .def("set", PyCUDATensorSetFromArray<uint16_t>)
F
fengjiayi 已提交
376
      .def("set", PyCUDATensorSetFromArray<uint8_t>)
Q
qingqing01 已提交
377
      .def("set", PyCUDATensorSetFromArray<int8_t>)
C
chengduoZH 已提交
378 379 380 381 382 383
      .def("set", PyCUDAPinnedTensorSetFromArray<float>)
      .def("set", PyCUDAPinnedTensorSetFromArray<int>)
      .def("set", PyCUDAPinnedTensorSetFromArray<double>)
      .def("set", PyCUDAPinnedTensorSetFromArray<int64_t>)
      .def("set", PyCUDAPinnedTensorSetFromArray<bool>)
      .def("set", PyCUDAPinnedTensorSetFromArray<uint16_t>)
F
fengjiayi 已提交
384
      .def("set", PyCUDAPinnedTensorSetFromArray<uint8_t>)
Q
qingqing01 已提交
385
      .def("set", PyCUDAPinnedTensorSetFromArray<int8_t>)
Q
qijun 已提交
386
#endif
387
      .def("shape", [](Tensor &self) { return vectorize(self.dims()); })
Y
yuyang18 已提交
388 389 390 391
      .def("_set_float_element", TensorSetElement<float>)
      .def("_get_float_element", TensorGetElement<float>)
      .def("_set_double_element", TensorSetElement<double>)
      .def("_get_double_element", TensorGetElement<double>)
X
xuezhong 已提交
392
      .def("_place", [](Tensor &self) { return self.place(); })
W
wopeizl 已提交
393 394
      .def("_dtype", [](Tensor &self) { return self.type(); })
      .def("__getitem__", PySliceTensor, py::return_value_policy::reference);
Y
Yu Yang 已提交
395

X
Xin Pan 已提交
396 397 398 399 400 401 402 403 404
  py::class_<LoDTensor, Tensor>(m, "LoDTensor", R"DOC(
    LoDTensor is a Tensor with optional LoD information.

    np.array(lod_tensor) can convert LoDTensor to numpy array.
    lod_tensor.lod() can retrieve the LoD information.

    LoD is short for Level of Details and is usually used for varied sequence
    length. You can skip the following comment if you don't need optional LoD.

Z
Zeng Jinle 已提交
405 406 407
    For example, a LoDTensor X can look like the example below. It contains 
    2 sequences. The first has length 2 and the second has length 3, as 
    described by x.lod.
X
Xin Pan 已提交
408

Z
Zeng Jinle 已提交
409 410 411
    The first tensor dimension 5=2+3 is calculated from LoD if it's available.
    It means the total number of sequence element. In X, each element has 2
    columns, hence [5, 2].
X
Xin Pan 已提交
412

Z
Zeng Jinle 已提交
413 414 415
    x.lod  = [[2, 3]]
     
    x.data = [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]]
X
Xin Pan 已提交
416

Z
Zeng Jinle 已提交
417
    x.shape = [5, 2]
X
Xin Pan 已提交
418

Z
Zeng Jinle 已提交
419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435
    LoD can have multiple levels (for example, a paragraph can have multiple
    sentences and a sentence can have multiple words). In the following
    LodTensor Y, the lod_level is 2. It means there are 2 sequence, the
    first sequence length is 2 (has 2 sub-sequences), the second one's
    length is 1. The first sequence's 2 sub-sequences have length 2 and 2,
    respectively. And the second sequence's 1 sub-sequence has length 3.

    y.lod = [[2 1], [2 2 3]]

    y.shape = [2+2+3, ...]

    Examples:
        .. code-block:: python

          import paddle.fluid as fluid

          t = fluid.LoDTensor()
X
Xin Pan 已提交
436 437 438 439 440 441 442 443 444 445 446 447

  Note:
      In above description, LoD is length-based. In Paddle internal
      implementation, lod is offset-based. Hence, internally,
      y.lod is represented as [[0, 2, 3], [0, 2, 4, 7]] (length-based
      equivlent would be [[2-0, 3-2], [2-0, 4-2, 7-4]]).

      Sometimes LoD is called recursive_sequence_length to be more
      self-explanatory. In this case, it must be length-based. Due to history
      reasons. when LoD is called lod in public API, it might be offset-based.
      Users should be careful about it.
        )DOC")
448
      .def("__array__", [](Tensor &self) { return TensorToPyArray(self); })
449 450 451 452 453 454 455 456 457 458 459 460 461 462
      .def("__init__",
           [](LoDTensor &instance, const std::vector<std::vector<size_t>>
                                       &recursive_sequence_lengths) {
             LoD new_lod;
             new_lod.reserve(recursive_sequence_lengths.size());
             std::copy(recursive_sequence_lengths.begin(),
                       recursive_sequence_lengths.end(),
                       std::back_inserter(new_lod));
             LoD new_offset_lod = ConvertToOffsetBasedLoD(new_lod);
             PADDLE_ENFORCE(
                 CheckLoD(new_offset_lod, -1),
                 "the provided recursive_sequence_lengths info is invalid");
             new (&instance) LoDTensor(new_offset_lod);
           })
Y
Yu Yang 已提交
463
      .def("__init__", [](LoDTensor &instance) { new (&instance) LoDTensor(); })
G
gongweibao 已提交
464 465 466 467 468
      // We implement offset based LOD in C++ while we use length based with
      // Python API. So we changed set_lod to set_recursive_sequence_lengths to
      // avoid misuse.
      // The discussion is here:
      // https://github.com/PaddlePaddle/Paddle/issues/10855
D
dangqingqing 已提交
469
      .def("set_lod",
470
           [](LoDTensor &self, const std::vector<std::vector<size_t>> &lod) {
471
             // the input lod is offset-based level-of-detail info
Y
Yu Yang 已提交
472
             LoD new_lod;
473 474
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
475 476
             PADDLE_ENFORCE(CheckLoD(new_lod, vectorize(self.dims()).front()),
                            "the provided lod info is invalid");
477
             self.set_lod(new_lod);
S
sneaxiy 已提交
478 479 480 481 482 483
           },
           py::arg("lod"), R"DOC(
           Set LoD of the LoDTensor.

           Args:
               lod (List[List[int]]): the lod to be set.
Z
Zeng Jinle 已提交
484 485 486 487 488 489 490 491 492 493

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_lod([[0, 2, 5]])
S
sneaxiy 已提交
494
           )DOC")
495 496 497 498 499 500 501 502 503 504 505 506 507 508 509
      .def("set_recursive_sequence_lengths",
           [](LoDTensor &self, const std::vector<std::vector<size_t>>
                                   &recursive_sequence_lengths) {
             // the input recursive_sequence_lengths is length-based
             // level-of-detail info
             LoD new_lod;
             new_lod.reserve(recursive_sequence_lengths.size());
             std::copy(recursive_sequence_lengths.begin(),
                       recursive_sequence_lengths.end(),
                       std::back_inserter(new_lod));
             LoD new_offset_lod = ConvertToOffsetBasedLoD(new_lod);
             PADDLE_ENFORCE(
                 CheckLoD(new_offset_lod, vectorize(self.dims()).front()),
                 "the provided recursive_sequence_lengths info is invalid");
             self.set_lod(new_offset_lod);
S
sneaxiy 已提交
510 511 512 513
           },
           py::arg("recursive_sequence_lengths"), R"DOC(
           Set LoD of the LoDTensor according to recursive sequence length.

S
sneaxiy 已提交
514
           For example, if recursive_sequence_lengths=[[2, 3]], meaning that
515 516
           there are two sequences with length 2 and 3 respectively, the
           corresponding lod would be [[0, 2, 2+3]], i.e, [[0, 2, 5]].
S
sneaxiy 已提交
517 518

           Args:
519
                recursive_sequence_lengths (List[List[int]]): sequence lengths.
Z
Zeng Jinle 已提交
520 521 522 523 524 525 526 527 528 529

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_recursive_sequence_lengths([[2, 3]])
S
sneaxiy 已提交
530
           )DOC")
531 532 533 534 535 536 537 538
      .def("lod",
           [](LoDTensor &self) -> std::vector<std::vector<size_t>> {
             // output the offset-based lod info
             LoD lod = self.lod();
             std::vector<std::vector<size_t>> new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             return new_lod;
S
sneaxiy 已提交
539 540 541 542 543 544
           },
           R"DOC(
           Return the LoD of the LoDTensor.

           Returns:
               out (List[List[int]]): the lod of the LoDTensor.
Z
Zeng Jinle 已提交
545 546 547 548 549 550 551 552 553 554 555

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_lod([[0, 2, 5]])
                 print(t.lod()) # [[0, 2, 5]]
S
sneaxiy 已提交
556
           )DOC")
G
gongweibao 已提交
557
      // Set above comments of set_lod.
558 559 560 561 562 563 564 565
      .def("recursive_sequence_lengths",
           [](LoDTensor &self) -> std::vector<std::vector<size_t>> {
             // output the length-based lod info
             LoD lod = ConvertToLengthBasedLoD(self.lod());
             std::vector<std::vector<size_t>> new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             return new_lod;
S
sneaxiy 已提交
566 567 568 569 570
           },
           R"DOC(
           Return the sequence length of the LoDTensor corresponding to LoD.

           Returns:
571
               out (List[List[int]): the sequence lengths.
Z
Zeng Jinle 已提交
572 573 574 575 576 577 578 579 580 581 582

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_recursive_sequence_lengths([[2, 3]])
                 print(t.recursive_sequence_lengths()) # [[2, 3]]
S
sneaxiy 已提交
583 584 585 586 587 588 589 590 591 592 593 594
           )DOC")
      .def("has_valid_recursive_sequence_lengths",
           [](LoDTensor &self) -> bool {
             // Check that the lod info is valid and match the outermost
             // dimension of the LoDTensor data
             return CheckLoD(self.lod(), vectorize(self.dims()).front());
           },
           R"DOC(
           Check whether the lod of the LoDTensor is valid.

           Returns:
               out (bool): whether the lod is valid.
Z
Zeng Jinle 已提交
595 596 597 598 599 600 601 602 603 604 605

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_recursive_sequence_lengths([[2, 3]])
                 print(t.has_valid_recursive_sequence_lengths()) # True
W
wopeizl 已提交
606 607 608 609 610 611 612
           )DOC")
      .def("__getitem__", PySliceTensor, py::return_value_policy::reference,
           R"DOC(
           Slice the original Tensor, and remove the LoD information.

           Returns:
               out (Tensor): new Tensor(NOT LoDTensor).
S
sneaxiy 已提交
613
           )DOC");
D
dangqingqing 已提交
614

Q
qijun 已提交
615 616 617 618 619 620 621 622 623 624 625
  py::class_<SelectedRows>(m, "SelectedRows")
      .def("__init__",
           [](SelectedRows &instance) { new (&instance) SelectedRows(); })
      .def("__init__",
           [](SelectedRows &instance, const std::vector<int64_t> rows,
              const int64_t &height) {
             new (&instance) SelectedRows(rows, height);
           })
      .def("get_tensor",
           [](SelectedRows &self) { return self.mutable_value(); },
           py::return_value_policy::reference)
626 627
      .def("numel",
           [](SelectedRows &self) -> int64_t { return self.value().numel(); })
Q
qijun 已提交
628 629
      .def("set_height", &SelectedRows::set_height)
      .def("height", &SelectedRows::height)
Q
qijun 已提交
630 631 632 633 634 635 636 637 638
      .def("set_rows",
           [](SelectedRows &self, std::vector<int64_t> rows) {
#ifndef PADDLE_WITH_CUDA
             self.set_rows(rows);
#else
        Vector<int64_t> new_rows(rows);
        self.set_rows(new_rows);
#endif
           })
639
      .def("sync_index", [](SelectedRows &instance) { instance.SyncIndex(); })
640
      .def("rows", [](SelectedRows &self) {
641 642 643 644 645
        auto rows = self.rows();
        std::vector<int64_t> new_rows;
        new_rows.reserve(rows.size());
        std::copy(rows.begin(), rows.end(), std::back_inserter(new_rows));
        return new_rows;
646
      });
Q
qijun 已提交
647

648
  py::class_<Variable>(m, "Variable", R"DOC(Variable Class.
649 650 651

All parameter, weight, gradient are variables in Paddle.
)DOC")
S
sneaxiy 已提交
652
      .def(py::init<>())
653
      .def("is_int", [](const Variable &var) { return var.IsType<int>(); })
654
      .def("set_int",
655 656
           [](Variable &var, int val) -> void { *var.GetMutable<int>() = val; })
      .def("get_int", [](const Variable &var) -> int { return var.Get<int>(); })
657 658 659 660 661 662 663
      .def("is_float", [](const Variable &var) { return var.IsType<float>(); })
      .def("set_float",
           [](Variable &var, float val) -> void {
             *var.GetMutable<float>() = val;
           })
      .def("get_float",
           [](const Variable &var) -> float { return var.Get<float>(); })
Y
Yu Yang 已提交
664
      .def("get_tensor",
665 666
           [](Variable &self) -> LoDTensor * {
             return self.GetMutable<LoDTensor>();
D
dangqingqing 已提交
667 668
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
669 670 671
      .def("get_lod_rank_table",
           [](Variable &self) { return self.GetMutable<LoDRankTable>(); },
           py::return_value_policy::reference)
Q
qijun 已提交
672 673 674 675 676
      .def("get_selected_rows",
           [](Variable &self) -> SelectedRows * {
             return self.GetMutable<SelectedRows>();
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
677 678 679
      .def("get_lod_tensor_array",
           [](Variable &self) { return self.GetMutable<LoDTensorArray>(); },
           py::return_value_policy::reference)
P
peizhilin 已提交
680
#if (defined(PADDLE_WITH_CUDA) && !defined(_WIN32))
D
Dong Zhihong 已提交
681 682 683 684 685
      .def("get_communicator",
           [](Variable &self) -> platform::Communicator * {
             return self.GetMutable<platform::Communicator>();
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
686
#endif
Y
Refine  
Yu Yang 已提交
687 688 689 690 691
      .def("get_reader",
           [](Variable &self) -> framework::ReaderHolder * {
             PADDLE_ENFORCE(self.IsType<framework::ReaderHolder>());
             return self.GetMutable<framework::ReaderHolder>();
           },
W
wopeizl 已提交
692
           py::return_value_policy::reference);
693

S
sneaxiy 已提交
694
  BindReader(&m);
Y
Refine  
Yu Yang 已提交
695

S
sneaxiy 已提交
696 697 698 699
  using LoDTensorBlockingQueue =
      ::paddle::operators::reader::LoDTensorBlockingQueue;
  using LoDTensorBlockingQueueHolder =
      ::paddle::operators::reader::LoDTensorBlockingQueueHolder;
S
sneaxiy 已提交
700

S
sneaxiy 已提交
701 702
  py::class_<LoDTensorBlockingQueue, std::shared_ptr<LoDTensorBlockingQueue>>(
      m, "LoDTensorBlockingQueue", "")
S
sneaxiy 已提交
703
      .def("push",
S
sneaxiy 已提交
704
           [](LoDTensorBlockingQueue &self,
S
sneaxiy 已提交
705
              const std::vector<framework::LoDTensor> &lod_tensor_vec) {
S
sneaxiy 已提交
706
             pybind11::gil_scoped_release release;
S
sneaxiy 已提交
707
             return self.Push(lod_tensor_vec);
S
sneaxiy 已提交
708
           })
S
sneaxiy 已提交
709 710 711 712
      .def("size", &LoDTensorBlockingQueue::Size)
      .def("capacity", &LoDTensorBlockingQueue::Cap)
      .def("close", &LoDTensorBlockingQueue::Close)
      .def("is_closed", &LoDTensorBlockingQueue::IsClosed);
S
sneaxiy 已提交
713

S
sneaxiy 已提交
714
  m.def("init_lod_tensor_blocking_queue",
Q
Qiao Longfei 已提交
715 716
        [](Variable &var,
           size_t capacity) -> std::shared_ptr<LoDTensorBlockingQueue> {
Q
Qiao Longfei 已提交
717
          VLOG(1) << "init_lod_tensor_blocking_queue";
Q
Qiao Longfei 已提交
718 719 720 721
          auto *holder = var.GetMutable<LoDTensorBlockingQueueHolder>();
          holder->InitOnce(capacity, FLAGS_reader_queue_speed_test_mode);
          return holder->GetQueue();
        },
S
sneaxiy 已提交
722
        py::return_value_policy::copy);
S
sneaxiy 已提交
723

S
sneaxiy 已提交
724
  py::class_<Scope>(m, "_Scope", R"DOC(
Q
Qiao Longfei 已提交
725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743
    Scope is an association of a name to Variable. All variables belong to Scope.

    Variables in a parent scope can be retrieved from local scope.

    You need to specify a scope to run a Net, i.e., `exe.Run(&scope)`.
    One net can run in different scopes and update different variable in the
    scope.

    You can create var in a scope and get it from the scope.

    Examples:
        .. code-block:: python

          # create tensor from a scope and set value to it.
          param = scope.var('Param').get_tensor()
          param_array = np.full((height, row_numel), 5.0).astype("float32")
          param.set(param_array, place)

        )DOC")
S
sneaxiy 已提交
744 745
      .def("_remove_from_pool",
           [](Scope &self) { ScopePool::Instance().Remove(&self); })
D
dongzhihong 已提交
746
      .def("var",
747
           [](Scope &self, const std::string &name) -> Variable * {
D
dongzhihong 已提交
748
             return self.Var(name);
Y
Yu Yang 已提交
749
           },
S
sneaxiy 已提交
750 751
           py::arg("name"),
           R"DOC(
752
           Find or create variable named :code:`name` in the current scope.
S
sneaxiy 已提交
753

754
           If the variable named :code:`name` does not exist in the
S
sneaxiy 已提交
755
           current scope, the variable would be created. Otherwise,
756
           return the existing variable.
S
sneaxiy 已提交
757 758

           Args:
759 760
               name (str): the variable name.

S
sneaxiy 已提交
761
           Returns:
762
               out (core.Variable): the found or created variable.
S
sneaxiy 已提交
763 764 765 766
           )DOC",
           py::return_value_policy::reference)
      .def("find_var", &Scope::FindVar, py::arg("name"),
           R"DOC(
767
           Find variable named :code:`name` in the current scope or
S
sneaxiy 已提交
768
           its parent scope. Return None if not found.
769

S
sneaxiy 已提交
770 771
           Args:
               name (str): the variable name.
772

S
sneaxiy 已提交
773
           Returns:
774
               out (core.Variable|None): the found variable or None.
S
sneaxiy 已提交
775
           )DOC",
776
           py::return_value_policy::reference)
777
      .def("new_scope", [](Scope &self) -> Scope * { return &self.NewScope(); },
S
sneaxiy 已提交
778 779 780 781 782 783
           R"DOC(
           Create a new sub-scope of the current scope.

           Returns:
               out (core._Scope): the created sub-scope.
           )DOC",
784
           py::return_value_policy::reference)
S
sneaxiy 已提交
785 786 787
      .def("drop_kids", &Scope::DropKids,
           R"DOC(
           Delete all sub-scopes of the current scope.
S
sneaxiy 已提交
788 789
           )DOC")
      .def("_kids", &Scope::kids);
790

S
sneaxiy 已提交
791 792 793 794 795 796
  m.def("Scope",
        []() -> Scope * {
          auto *s = new Scope();
          ScopePool::Instance().Insert(std::unique_ptr<Scope>(s));
          return s;
        },
S
sneaxiy 已提交
797 798
        R"DOC(
        Create a new scope.
799

S
sneaxiy 已提交
800 801 802
        Returns:
            out (core._Scope): the created scope.
        )DOC",
S
sneaxiy 已提交
803 804
        py::return_value_policy::reference);

Y
Yu Yang 已提交
805 806
  //! @note: Be careful! PyBind will return std::string as an unicode, not
  //! Python str. If you want a str object, you should cast them in Python.
Y
Yu Yang 已提交
807 808
  m.def("get_all_op_protos", []() -> std::vector<py::bytes> {
    std::vector<py::bytes> ret_values;
809 810 811 812 813 814 815 816 817 818
    for (auto &iter : OpInfoMap::Instance().map()) {
      auto &info = iter.second;
      if (info.HasOpProtoAndChecker()) {
        std::string str;
        PADDLE_ENFORCE(
            info.Proto().SerializeToString(&str),
            "Serialize OpProto Error. This could be a bug of Paddle.");
        ret_values.emplace_back(str);
      }
    }
Y
Yu Yang 已提交
819 820
    return ret_values;
  });
821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836
  m.def(
      "get_grad_op_desc", [](const OpDesc &op_desc,
                             const std::unordered_set<std::string> &no_grad_set,
                             const std::vector<BlockDesc *> &grad_sub_block) {
        std::unordered_map<std::string, std::string> grad_to_var;
        std::vector<std::unique_ptr<OpDesc>> grad_op_descs =
            framework::OpInfoMap::Instance()
                .Get(op_desc.Type())
                .GradOpMaker()(op_desc, no_grad_set, &grad_to_var,
                               grad_sub_block);
        std::vector<OpDesc *> grad_op_desc_ptrs(grad_op_descs.size());
        std::transform(grad_op_descs.begin(), grad_op_descs.end(),
                       grad_op_desc_ptrs.begin(),
                       [](std::unique_ptr<OpDesc> &p) { return p.release(); });
        return std::make_pair(grad_op_desc_ptrs, grad_to_var);
      });
Y
Yu Yang 已提交
837
  m.def("prune", [](const ProgramDesc &origin,
838
                    const std::vector<std::array<size_t, 2>> &targets) {
Y
Yu Yang 已提交
839
    ProgramDesc prog_with_targets(origin);
840
    for (const auto &t : targets) {
841
      prog_with_targets.MutableBlock(t[0])->Op(t[1])->SetIsTarget(true);
842
    }
843
    proto::ProgramDesc pruned_desc;
844
    Prune(*prog_with_targets.Proto(), &pruned_desc);
Y
Yu Yang 已提交
845
    return new ProgramDesc(pruned_desc);
846
  });
847 848 849 850
  m.def("empty_var_name",
        []() { return std::string(framework::kEmptyVarName); });
  m.def("grad_var_suffix",
        []() { return std::string(framework::kGradVarSuffix); });
851 852 853
  m.def_submodule(
       "var_names",
       "The module will return special predefined variable name in Paddle")
Y
Yi Wang 已提交
854 855
      .def("empty", []() { return kEmptyVarName; })
      .def("temp", []() { return kTempVarName; });
Q
qijun 已提交
856
  // clang-format off
Y
Yu Yang 已提交
857
  py::class_<paddle::platform::DeviceContext>(m, "DeviceContext")
Q
qijun 已提交
858 859
      .def_static("create",
                  [](paddle::platform::CPUPlace& place)
Q
qijun 已提交
860
                      -> paddle::platform::DeviceContext* {
Q
qijun 已提交
861 862 863
                    return new paddle::platform::CPUDeviceContext();
                  })
      .def_static("create",
D
dzhwinter 已提交
864
                  [](paddle::platform::CUDAPlace& place)
Q
qijun 已提交
865
                      -> paddle::platform::DeviceContext* {
866
#ifndef PADDLE_WITH_CUDA
D
dzhwinter 已提交
867
                    PADDLE_THROW("CUDAPlace is not supported in CPU device.");
Q
qijun 已提交
868
#else
Q
qijun 已提交
869
                    return new paddle::platform::CUDADeviceContext(place);
Q
qijun 已提交
870
#endif
C
chengduoZH 已提交
871 872 873 874 875 876 877 878 879 880 881
                  })
          .def_static("create",
                [](paddle::platform::CUDAPinnedPlace& place)
                        -> paddle::platform::DeviceContext* {
#ifndef PADDLE_WITH_CUDA
                  PADDLE_THROW(
                        "CUDAPinnedPlace is not supported in CPU device.");
#else
                  return new paddle::platform::CUDAPinnedDeviceContext(place);
#endif
                });;
D
Dong Zhihong 已提交
882
// clang-format on
P
peizhilin 已提交
883
#if (defined(PADDLE_WITH_CUDA) && !defined(_WIN32))
D
Dong Zhihong 已提交
884 885
  py::class_<platform::Communicator>(m, "Communicator").def(py::init<>());
#endif
886 887 888 889
  py::class_<platform::CUDAPlace>(m, "CUDAPlace", R"DOC(
    CUDAPlace is a descriptor of a device. It represents a GPU, and each CUDAPlace
    has a dev_id to indicate the number of cards represented by the current CUDAPlace.
    The memory of CUDAPlace with different dev_id is not accessible.
L
lujun 已提交
890 891 892 893 894 895

    Examples:
        .. code-block:: python

          gpu_place = fluid.CUDAPlace(0)

896
        )DOC")
S
sneaxiy 已提交
897 898 899 900 901 902 903 904 905 906 907 908
      .def("__init__",
           [](platform::CUDAPlace &self, int dev_id) {
#ifdef PADDLE_WITH_CUDA
             PADDLE_ENFORCE(
                 dev_id >= 0 && dev_id < platform::GetCUDADeviceCount(),
                 "Invalid CUDAPlace(%d), must inside [0, %d)", dev_id,
                 platform::GetCUDADeviceCount());
             new (&self) platform::CUDAPlace(dev_id);
#else
             PADDLE_THROW("Cannot use CUDAPlace in CPU only version");
#endif
           })
S
sneaxiy 已提交
909 910 911 912 913 914
      .def("_type", &PlaceIndex<platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::Place>)
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::CPUPlace>)
      .def("_equals",
           &IsSamePlace<platform::CUDAPlace, platform::CUDAPinnedPlace>)
D
dzhwinter 已提交
915
      .def("__str__", string::to_string<const platform::CUDAPlace &>);
Q
qijun 已提交
916

917 918 919
  py::class_<paddle::platform::CPUPlace>(m, "CPUPlace", R"DOC(
    CPUPlace is a descriptor of a device. It represents a CPU, and the memory
    CPUPlace can be accessed by CPU.
L
lujun 已提交
920 921 922 923 924 925

    Examples:
        .. code-block:: python

          cpu_place = fluid.CPUPlace()

926
        )DOC")
927
      .def(py::init<>())
S
sneaxiy 已提交
928 929 930 931 932 933
      .def("_type", &PlaceIndex<platform::CPUPlace>)
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::Place>)
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::CPUPlace>)
      .def("_equals",
           &IsSamePlace<platform::CPUPlace, platform::CUDAPinnedPlace>)
934
      .def("__str__", string::to_string<const platform::CPUPlace &>);
Y
Yu Yang 已提交
935

936 937 938
  py::class_<paddle::platform::CUDAPinnedPlace>(m, "CUDAPinnedPlace", R"DOC(
    CUDAPinnedPlace is a descriptor of a device. The memory of CUDAPinnedPlace
    can be accessed by GPU and CPU.
L
lujun 已提交
939 940 941 942 943 944

    Examples:
        .. code-block:: python

          place = fluid.CUDAPinnedPlace()

945
        )DOC")
S
sneaxiy 已提交
946
      .def("__init__",
S
sneaxiy 已提交
947
           [](platform::CUDAPinnedPlace &self) {
S
sneaxiy 已提交
948 949 950
#ifndef PADDLE_WITH_CUDA
             PADDLE_THROW("Cannot use CUDAPinnedPlace in CPU only version");
#endif
S
sneaxiy 已提交
951
             new (&self) platform::CUDAPinnedPlace();
S
sneaxiy 已提交
952
           })
S
sneaxiy 已提交
953 954 955 956 957 958 959 960
      .def("_type", &PlaceIndex<platform::CUDAPinnedPlace>)
      .def("_equals", &IsSamePlace<platform::CUDAPinnedPlace, platform::Place>)
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::CUDAPlace>)
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::CPUPlace>)
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::CUDAPinnedPlace>)
C
chengduoZH 已提交
961 962
      .def("__str__", string::to_string<const platform::CUDAPinnedPlace &>);

Y
Yu Yang 已提交
963 964
  py::class_<platform::Place>(m, "Place")
      .def(py::init<>())
S
sneaxiy 已提交
965 966 967 968 969
      .def("_type", &PlaceIndex<platform::Place>)
      .def("_equals", &IsSamePlace<platform::Place, platform::Place>)
      .def("_equals", &IsSamePlace<platform::Place, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::Place, platform::CPUPlace>)
      .def("_equals", &IsSamePlace<platform::Place, platform::CUDAPinnedPlace>)
X
xuezhong 已提交
970 971
      .def("is_gpu_place",
           [](platform::Place &self) { return platform::is_gpu_place(self); })
S
sneaxiy 已提交
972 973 974 975 976 977
      .def("is_cpu_place",
           [](platform::Place &self) { return platform::is_cpu_place(self); })
      .def("is_cuda_pinned_place",
           [](platform::Place &self) {
             return platform::is_cuda_pinned_place(self);
           })
X
xuezhong 已提交
978 979 980 981
      .def("gpu_device_id",
           [](platform::Place &self) {
             return boost::get<platform::CUDAPlace>(self).device;
           })
S
sneaxiy 已提交
982 983
      .def("set_place", [](platform::Place &self,
                           const platform::Place &other) { self = other; })
Y
Yu Yang 已提交
984 985 986 987 988
      .def("set_place",
           [](platform::Place &self, const platform::CPUPlace &cpu_place) {
             self = cpu_place;
           })
      .def("set_place",
D
dzhwinter 已提交
989
           [](platform::Place &self, const platform::CUDAPlace &gpu_place) {
Y
Yu Yang 已提交
990
             self = gpu_place;
C
chengduoZH 已提交
991 992
           })
      .def("set_place", [](platform::Place &self,
C
chengduoZH 已提交
993 994
                           const platform::CUDAPinnedPlace &cuda_pinned_place) {
        self = cuda_pinned_place;
C
chengduoZH 已提交
995
      });
Y
Yu Yang 已提交
996

Y
Yu Yang 已提交
997 998 999
  py::class_<OperatorBase>(m, "Operator")
      .def_static("create",
                  [](py::bytes protobin) {
1000
                    proto::OpDesc desc;
Y
Yu Yang 已提交
1001 1002 1003 1004 1005
                    PADDLE_ENFORCE(desc.ParsePartialFromString(protobin),
                                   "Cannot parse user input to OpDesc");
                    PADDLE_ENFORCE(desc.IsInitialized(),
                                   "User OpDesc is not initialized, reason %s",
                                   desc.InitializationErrorString());
1006
                    return OpRegistry::CreateOp(desc);
Y
Yu Yang 已提交
1007
                  })
1008
      .def("run",
1009
           [](OperatorBase &self, const Scope &scope,
D
dzhwinter 已提交
1010 1011 1012
              const platform::CPUPlace &place) { self.Run(scope, place); })
      .def("run",
           [](OperatorBase &self, const Scope &scope,
D
dzhwinter 已提交
1013
              const platform::CUDAPlace &place) { self.Run(scope, place); })
C
chengduoZH 已提交
1014 1015 1016 1017 1018
      .def("run",
           [](OperatorBase &self, const Scope &scope,
              const platform::CUDAPinnedPlace &place) {
             self.Run(scope, place);
           })
Y
Yu Yang 已提交
1019 1020 1021 1022 1023 1024 1025
      .def("type",
           [](const OperatorBase &op) -> std::string { return op.Type(); })
      .def("outputs",
           [](const OperatorBase &op)
               -> std::map<std::string, std::vector<std::string>> {
                 return op.Outputs();
               })
Q
qijun 已提交
1026 1027
      .def("output_vars",
           [](const OperatorBase &op) { return op.OutputVars(true); })
Y
Yu Yang 已提交
1028
      .def("inputs", [](const OperatorBase &op) { return op.Inputs(); })
Q
qijun 已提交
1029
      .def("input_vars", [](const OperatorBase &op) { return op.InputVars(); })
Y
Yu Yang 已提交
1030 1031 1032 1033
      .def("__str__", &OperatorBase::DebugString)
      .def("no_intermediate_outputs",
           [](const OperatorBase &op) { return op.OutputVars(false); })
      .def("support_gpu", &OperatorBase::SupportGPU);
Y
Yu Yang 已提交
1034

1035 1036 1037
  py::class_<framework::ExecutorPrepareContext>(m, "ExecutorPrepareContext")
      .def(py::init<const ProgramDesc &, size_t>());

F
fengjiayi 已提交
1038
  py::class_<framework::Executor>(m, "Executor")
D
dzhwinter 已提交
1039
      .def(py::init<const platform::Place &>())
Y
Yancey1989 已提交
1040
      .def("close", &Executor::Close)
1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056
      .def("run_from_dataset", &Executor::RunFromDataset,
           py::call_guard<py::gil_scoped_release>())
      .def("run_prepared_ctx",
           [](Executor &self, ExecutorPrepareContext *ctx, Scope *scope,
              std::map<std::string, const LoDTensor *> *feed_targets,
              std::map<std::string, LoDTensor *> *fetch_targets,
              bool create_local_scope = true, bool create_vars = true,
              const std::string &feed_holder_name = "feed",
              const std::string &fetch_holder_name = "fetch") {
             pybind11::gil_scoped_release release;
             self.RunPreparedContext(ctx, scope, feed_targets, fetch_targets,
                                     create_local_scope, create_vars,
                                     feed_holder_name, fetch_holder_name);
           })
      .def("prepare_ctx_cache", &Executor::PrepareCtxCache,
           py::call_guard<py::gil_scoped_release>())
S
sneaxiy 已提交
1057
      .def("run", [](Executor &self, const ProgramDesc &prog, Scope *scope,
S
sneaxiy 已提交
1058 1059
                     int block_id, bool create_local_scope, bool create_vars,
                     const std::vector<std::string> &fetch_vars) {
S
sneaxiy 已提交
1060
        pybind11::gil_scoped_release release;
S
sneaxiy 已提交
1061 1062
        self.Run(prog, scope, block_id, create_local_scope, create_vars,
                 fetch_vars);
S
sneaxiy 已提交
1063
      });
S
sneaxiy 已提交
1064

D
dzhwinter 已提交
1065
  m.def("init_gflags", framework::InitGflags);
Y
Yang Yu 已提交
1066
  m.def("init_glog", framework::InitGLOG);
1067
  m.def("init_dgc", framework::InitDGC);
X
Xin Pan 已提交
1068 1069
  m.def("init_devices",
        [](bool init_p2p) { framework::InitDevices(init_p2p); });
1070

1071
  m.def("is_compiled_with_ngraph", IsCompiledWithNGRAPH);
1072
  m.def("is_compiled_with_cuda", IsCompiledWithCUDA);
1073
  m.def("is_compiled_with_mkldnn", IsCompiledWithMKLDNN);
1074
  m.def("is_compiled_with_brpc", IsCompiledWithBrpc);
Y
update  
Yancey1989 已提交
1075
  m.def("is_compiled_with_dist", IsCompiledWithDIST);
1076 1077 1078 1079 1080 1081
#ifdef PADDLE_WITH_CUDA
  m.def("is_float16_supported", [](const platform::CUDAPlace &place) -> bool {
    // Only GPUs with Compute Capability >= 53 support float16
    return platform::GetCUDAComputeCapability(place.device) >= 53;
  });
#endif
1082

1083
  m.def("set_feed_variable", framework::SetFeedVariable);
Q
qijun 已提交
1084
  m.def("get_fetch_variable", framework::GetFetchVariable);
1085
  m.def("get_variable_tensor", framework::GetVariableTensor);
Q
qijun 已提交
1086

X
Xin Pan 已提交
1087 1088
  m.def("_is_program_version_supported", IsProgramVersionSupported);

1089 1090 1091 1092 1093
  BindProgramDesc(&m);
  BindBlockDesc(&m);
  BindVarDsec(&m);
  BindOpDesc(&m);
  BindConstValue(&m);
Y
Yu Yang 已提交
1094

Y
Yu Yang 已提交
1095 1096 1097 1098 1099 1100 1101 1102 1103
  py::class_<framework::LoDRankTable>(m, "LodRankTable")
      .def("items", [](framework::LoDRankTable &table) {
        std::vector<std::pair<size_t, size_t>> res;
        for (auto &item : table.items()) {
          res.push_back({item.index, item.length});
        }
        return res;
      });

Z
Zeng Jinle 已提交
1104 1105 1106 1107 1108 1109 1110 1111 1112 1113
  py::class_<LoDTensorArray>(m, "LoDTensorArray", R"DOC(
    Array of LoDTensor.

    Examples:
        .. code-block:: python
        
          import paddle.fluid as fluid

          arr = fluid.LoDTensorArray()
)DOC")
S
sneaxiy 已提交
1114 1115
      .def("__init__",
           [](LoDTensorArray &instance) { new (&instance) LoDTensorArray(); })
Y
Yu Yang 已提交
1116 1117 1118 1119 1120 1121 1122 1123 1124 1125
      .def("__getitem__",
           [](LoDTensorArray &self, size_t i) { return &self.at(i); },
           py::return_value_policy::reference)
      .def("__len__", [](LoDTensorArray &self) { return self.size(); })
      .def("__setitem__",
           [](LoDTensorArray &self, size_t i, const LoDTensor &t) {
             PADDLE_ENFORCE_LT(i, self.size());
             self[i].ShareDataWith(t);
             self[i].set_lod(t.lod());
           })
S
sneaxiy 已提交
1126 1127 1128 1129 1130 1131
      .def("append",
           [](LoDTensorArray &self, const LoDTensor &t) {
             self.emplace_back();
             self.back().ShareDataWith(t);
             self.back().set_lod(t.lod());
           },
Z
Zeng Jinle 已提交
1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145
           py::arg("tensor"), R"DOC(
             Append a LoDensor to LoDTensorArray.

             Examples:
                 .. code-block:: python

                   import paddle.fluid as fluid
                   import numpy as np

                   arr = fluid.LoDTensorArray()
                   t = fluid.LoDTensor()
                   t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                   arr.append(t)
           )DOC");
Y
Yu Yang 已提交
1146

D
dzhwinter 已提交
1147 1148 1149
  m.def("IsInplace",
        [](std::string op) -> bool { return operators::IsInplace(op); });

Y
Yu Yang 已提交
1150
  m.def("op_support_gpu", OpSupportGPU);
D
Dong Zhihong 已提交
1151
#ifdef PADDLE_WITH_CUDA
D
Dong Zhihong 已提交
1152
  m.def("get_cuda_device_count", platform::GetCUDADeviceCount);
D
dangqingqing 已提交
1153

P
peizhilin 已提交
1154
#ifndef _WIN32
D
dangqingqing 已提交
1155 1156 1157
  m.def("nvprof_init", platform::CudaProfilerInit);
  m.def("nvprof_start", platform::CudaProfilerStart);
  m.def("nvprof_stop", platform::CudaProfilerStop);
D
Dong Zhihong 已提交
1158
#endif
P
peizhilin 已提交
1159
#endif
Y
Yu Yang 已提交
1160

1161 1162 1163 1164
  py::enum_<platform::ProfilerState>(m, "ProfilerState", py::arithmetic())
      .value("kDisabled", platform::ProfilerState::kDisabled)
      .value("kCPU", platform::ProfilerState::kCPU)
      .value("kCUDA", platform::ProfilerState::kCUDA)
1165
      .value("kAll", platform::ProfilerState::kAll)
1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178
      .export_values();

  py::enum_<platform::EventSortingKey>(m, "EventSortingKey", py::arithmetic())
      .value("kDefault", platform::EventSortingKey::kDefault)
      .value("kCalls", platform::EventSortingKey::kCalls)
      .value("kTotal", platform::EventSortingKey::kTotal)
      .value("kMin", platform::EventSortingKey::kMin)
      .value("kMax", platform::EventSortingKey::kMax)
      .value("kAve", platform::EventSortingKey::kAve)
      .export_values();

  m.def("enable_profiler", platform::EnableProfiler);
  m.def("disable_profiler", platform::DisableProfiler);
X
Xin Pan 已提交
1179
  m.def("is_profiler_enabled", platform::IsProfileEnabled);
1180
  m.def("reset_profiler", platform::ResetProfiler);
1181
  m.def("get_pass", [](const std::string &pass_type) {
W
WangZhen 已提交
1182 1183 1184
    auto pass = framework::ir::PassRegistry::Instance().Get(pass_type);
    return std::shared_ptr<framework::ir::Pass>(std::move(pass));
  });
Y
Yu Yang 已提交
1185

1186 1187
  py::class_<ir::Pass, std::shared_ptr<ir::Pass>> pass(m, "Pass");
  pass.def(py::init())
W
WangZhen 已提交
1188
      .def("has", &ir::Pass::Has)
1189 1190 1191
      .def("set_not_owned",
           [](ir::Pass &self, const std::string &attr_name, ProgramDesc &attr) {
             self.SetNotOwned<ProgramDesc>(attr_name, &attr);
W
WangZhen 已提交
1192
           })
1193
      .def(
1194
          "set",
1195 1196 1197
          [](ir::Pass &self, const std::string &name, const std::string &attr) {
            self.Set<std::string>(name, new std::string(attr));
          })
1198 1199
      .def("set", [](ir::Pass &self, const std::string &name,
                     int val) { self.Set<const int>(name, new int(val)); })
F
flame 已提交
1200 1201
      .def("type", &ir::Pass::Type)
      .def("apply", [](ir::Pass &self, std::shared_ptr<ir::Graph> graph) {
1202
        self.Apply(graph.get());
F
flame 已提交
1203
      });
1204

X
fix  
Xin Pan 已提交
1205 1206
  py::class_<ir::PassBuilder, std::shared_ptr<ir::PassBuilder>> pb(
      m, "PassBuilder");
1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220
  pb.def(py::init())
      .def("append_pass",
           [](ir::PassBuilder &self,
              const std::string &pass_type) -> std::shared_ptr<ir::Pass> {
             return self.AppendPass(pass_type);
           })
      .def("all_passes", [](ir::PassBuilder &self) { return self.AllPasses(); })
      .def("insert_pass",
           [](ir::PassBuilder &self, size_t idx, const std::string &pass_type) {
             return self.InsertPass(idx, pass_type);
           })
      .def("remove_pass",
           [](ir::PassBuilder &self, size_t idx) { self.RemovePass(idx); });

Y
yuyang18 已提交
1221
  // -- python binds for parallel executor.
X
Xin Pan 已提交
1222

Y
yuyang18 已提交
1223
  py::class_<ParallelExecutor> pe(m, "ParallelExecutor");
C
chengduo 已提交
1224 1225 1226 1227
  py::class_<ExecutionStrategy> exec_strategy(pe, "ExecutionStrategy", R"DOC(
    ExecutionStrategy allows the user to more preciously control how to run
    the program in ParallelExecutor by setting the property.

C
chengduo 已提交
1228 1229 1230
    Examples:
        .. code-block:: python

1231 1232 1233 1234 1235 1236 1237 1238 1239 1240
          x = fluid.layers.data(name='x', shape=[13], dtype='float32')
          y = fluid.layers.data(name='y', shape=[1], dtype='float32')
          y_predict = fluid.layers.fc(input=x, size=1, act=None)

          cost = fluid.layers.square_error_cost(input=y_predict, label=y)
          avg_loss = fluid.layers.mean(cost)

          sgd_optimizer = fluid.optimizer.SGD(learning_rate=0.001)
          sgd_optimizer.minimize(avg_loss)

C
chengduo 已提交
1241 1242 1243
          exec_strategy = fluid.ExecutionStrategy()
          exec_strategy.num_threads = 4

1244 1245
          train_exe = fluid.ParallelExecutor(use_cuda=False,
                                             loss_name=avg_loss.name,
C
chengduo 已提交
1246 1247
                                             exec_strategy=exec_strategy)

C
chengduo 已提交
1248 1249
        )DOC");

Y
yuyang18 已提交
1250
  exec_strategy.def(py::init())
Y
yuyang18 已提交
1251 1252 1253 1254 1255
      .def_property(
          "num_threads",
          [](const ExecutionStrategy &self) { return self.num_threads_; },
          [](ExecutionStrategy &self, size_t num_threads) {
            self.num_threads_ = num_threads;
C
chengduo 已提交
1256 1257 1258 1259 1260 1261 1262 1263 1264 1265
          },
          R"DOC(The type is INT, num_threads represents the size of thread pool that
            used to run the operators of the current program in ParallelExecutor.
            If :math:`num\_threads=1`, all the operators will execute one by one,
            but the order maybe difference between iterations.
            If it is not set, it will be set in ParallelExecutor according to the
            device type and device count, for GPU, :math:`num\_threads=device\_count*4`, for CPU,
            :math:`num\_threads=CPU\_NUM*4`, the explanation of:math:`CPU\_NUM` is in ParallelExecutor.
            if it is not set, ParallelExecutor will get the cpu count by calling
            `multiprocessing.cpu_count()`. Default 0.)DOC")
Y
yuyang18 已提交
1266
      .def_property(
1267 1268 1269 1270
          "use_cuda",
          [](const ExecutionStrategy &self) { return self.use_cuda_; },
          [](ExecutionStrategy &self, bool use_cuda) {
            self.use_cuda_ = use_cuda;
C
chengduo 已提交
1271 1272 1273 1274
          })  // FIXME(chengduo): Doesn't add doc for 'use_cuda', use_cuda may
      // make user confuse, because ParallelExecutor has a parameter named
      // 'use_cuda' too, in current implementation, ParallelExecutor's
      // 'use_cuda' will rewrite ExecutionStrategy's 'use_cuda'.
Y
yuyang18 已提交
1275 1276 1277 1278 1279
      .def_property(
          "allow_op_delay",
          [](const ExecutionStrategy &self) { return self.allow_op_delay_; },
          [](ExecutionStrategy &self, bool allow_op_delay) {
            self.allow_op_delay_ = allow_op_delay;
C
chengduo 已提交
1280 1281 1282 1283
          },
          R"DOC(The type is BOOL, allow_op_delay represents whether to delay the
                communication operators to run, it may make the execution faster.
                Note that in some models, allow_op_delay may cause program hang. Default False.)DOC")
Y
yuyang18 已提交
1284 1285 1286 1287 1288 1289 1290
      .def_property(
          "num_iteration_per_drop_scope",
          [](const ExecutionStrategy &self) {
            return self.num_iteration_per_drop_scope_;
          },
          [](ExecutionStrategy &self, size_t num_iteration_per_drop_scope) {
            self.num_iteration_per_drop_scope_ = num_iteration_per_drop_scope;
C
chengduo 已提交
1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301
          },
          R"DOC(The type is INT, num_iteration_per_drop_scope indicates how
                many iterations to clean up the temp variables which
                is generated during execution. It may make the execution faster,
                because the temp variable's shape maybe the same between two iterations. Default 100.

                NOTES:
                    1. If you fetch data when calling the 'run', the ParallelExecutor
                       will clean up the temp variables at the end of the current iteration.
                    2. In some NLP model, it may cause the GPU memory is insufficient,
                       in this case, you should reduce `num_iteration_per_drop_scope`.
1302
              )DOC")
Q
Qiao Longfei 已提交
1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313
      .def_property(
          "num_iteration_per_run",
          [](const ExecutionStrategy &self) {
            return self.num_iteration_per_run_;
          },
          [](ExecutionStrategy &self, size_t num_iteration_per_run) {
            self.num_iteration_per_run_ = num_iteration_per_run;
          },
          R"DOC(This config that how many iteration the executor will run when
                user call pe.run() in python
              )DOC")
1314 1315 1316 1317 1318
      .def_property("_dry_run",
                    [](const ExecutionStrategy &self) { return self.dry_run_; },
                    [](ExecutionStrategy &self, bool dry_run) {
                      self.dry_run_ = dry_run;
                    });
C
chengduo 已提交
1319

Y
yuyang18 已提交
1320
  exec_strategy.def_property(
Y
yuyang18 已提交
1321 1322 1323 1324 1325 1326 1327
      "use_experimental_executor",
      [](const ExecutionStrategy &self) {
        return self.type_ == ExecutionStrategy::kExperimental;
      },
      [](ExecutionStrategy &self, bool experimental) {
        self.type_ = experimental ? ExecutionStrategy::kExperimental
                                  : ExecutionStrategy::kDefault;
Y
yuyang18 已提交
1328 1329
      });

C
chengduo 已提交
1330 1331 1332 1333
  py::class_<BuildStrategy> build_strategy(pe, "BuildStrategy", R"DOC(
    BuildStrategy allows the user to more preciously control how to
    build the SSA Graph in ParallelExecutor by setting the property.

C
chengduo 已提交
1334 1335 1336
    Examples:
        .. code-block:: python

F
flame 已提交
1337 1338 1339
            import paddle.fluid as fluid
            build_strategy = fluid.BuildStrategy()
            build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce
C
chengduo 已提交
1340
)DOC");
Y
yuyang18 已提交
1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356

  py::enum_<BuildStrategy::ReduceStrategy>(build_strategy, "ReduceStrategy")
      .value("Reduce", BuildStrategy::ReduceStrategy::kReduce)
      .value("AllReduce", BuildStrategy::ReduceStrategy::kAllReduce);
  py::enum_<BuildStrategy::GradientScaleStrategy>(build_strategy,
                                                  "GradientScaleStrategy")
      .value("CoeffNumDevice",
             BuildStrategy::GradientScaleStrategy::kCoeffNumDevice)
      .value("One", BuildStrategy::GradientScaleStrategy::kOne)
      .value("Customized", BuildStrategy::GradientScaleStrategy::kCustomized);

  build_strategy.def(py::init())
      .def_property(
          "reduce_strategy",
          [](const BuildStrategy &self) { return self.reduce_; },
          [](BuildStrategy &self, BuildStrategy::ReduceStrategy strategy) {
X
Xin Pan 已提交
1357
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
Y
yuyang18 已提交
1358
            self.reduce_ = strategy;
C
chengduo 已提交
1359 1360
          },
          R"DOC(The type is STR, there are two reduce strategies in ParallelExecutor,
F
flame 已提交
1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373
                'AllReduce' and 'Reduce'. If you want that all the parameters'
                optimization are done on all devices independently, you should choose 'AllReduce';
                if you choose 'Reduce', all the parameters' optimization will be evenly distributed
                to different devices, and then broadcast the optimized parameter to other devices.
                In some models, `Reduce` is faster. Default 'AllReduce'.

                Examples:
                    .. code-block:: python

                        import paddle.fluid as fluid
                        build_strategy = fluid.BuildStrategy()
                        build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce
                  )DOC")
Y
yuyang18 已提交
1374 1375 1376 1377 1378
      .def_property(
          "gradient_scale_strategy",
          [](const BuildStrategy &self) { return self.gradient_scale_; },
          [](BuildStrategy &self,
             BuildStrategy::GradientScaleStrategy strategy) {
X
Xin Pan 已提交
1379
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
Y
yuyang18 已提交
1380
            self.gradient_scale_ = strategy;
C
chengduo 已提交
1381 1382
          },
          R"DOC(The type is STR, there are three ways of defining :math:`loss@grad` in
F
flame 已提交
1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394
                ParallelExecutor, 'CoeffNumDevice', 'One' and 'Customized'. By default,
                ParallelExecutor sets the :math:`loss@grad` according to the number of devices.
                If you want to customize :math:`loss@grad`, you can choose 'Customized'.
                Default 'CoeffNumDevice'.

                Examples:
                    .. code-block:: python

                        import paddle.fluid as fluid
                        build_strategy = fluid.BuildStrategy()
                        build_strategy.gradient_scale_strategy = True
                   )DOC")
Y
yuyang18 已提交
1395 1396 1397 1398
      .def_property(
          "debug_graphviz_path",
          [](const BuildStrategy &self) { return self.debug_graphviz_path_; },
          [](BuildStrategy &self, const std::string &path) {
X
Xin Pan 已提交
1399
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
Y
yuyang18 已提交
1400
            self.debug_graphviz_path_ = path;
C
chengduo 已提交
1401 1402
          },
          R"DOC(The type is STR, debug_graphviz_path indicate the path that
F
flame 已提交
1403 1404 1405 1406 1407 1408 1409 1410 1411 1412
                writing the SSA Graph to file in the form of graphviz.
                It is useful for debugging. Default ""

                Examples:
                    .. code-block:: python

                        import paddle.fluid as fluid
                        build_strategy = fluid.BuildStrategy()
                        build_strategy.debug_graphviz_path = ""
                    )DOC")
S
sneaxiy 已提交
1413 1414 1415 1416 1417 1418
      .def_property(
          "enable_sequential_execution",
          [](const BuildStrategy &self) {
            return self.enable_sequential_execution_;
          },
          [](BuildStrategy &self, bool b) {
X
Xin Pan 已提交
1419
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
S
sneaxiy 已提交
1420 1421
            self.enable_sequential_execution_ = b;
          },
F
flame 已提交
1422 1423 1424 1425 1426 1427 1428 1429 1430
          R"DOC(The type is BOOL. If set True, the execution order of ops would be the same as what is in the program. Default False.

                Examples:
                    .. code-block:: python

                        import paddle.fluid as fluid
                        build_strategy = fluid.BuildStrategy()
                        build_strategy.enable_sequential_execution = True
          )DOC")
S
sneaxiy 已提交
1431 1432 1433 1434 1435 1436
      .def_property(
          "remove_unnecessary_lock",
          [](const BuildStrategy &self) {
            return self.remove_unnecessary_lock_;
          },
          [](BuildStrategy &self, bool b) {
X
Xin Pan 已提交
1437
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
S
sneaxiy 已提交
1438 1439
            self.remove_unnecessary_lock_ = b;
          },
F
flame 已提交
1440 1441 1442 1443 1444 1445 1446 1447 1448
          R"DOC(The type is BOOL. If set True, some locks in GPU ops would be released and ParallelExecutor would run faster. Default True.

                Examples:
                    .. code-block:: python

                        import paddle.fluid as fluid
                        build_strategy = fluid.BuildStrategy()
                        build_strategy.remove_unnecessary_lock = True
          )DOC")
1449 1450 1451 1452 1453 1454
      .def_property(
          "num_trainers",
          [](const BuildStrategy &self) { return self.num_trainers_; },
          [](BuildStrategy &self, int num_trainers) {
            self.num_trainers_ = num_trainers;
          })
1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466
      .def_property(
          "trainers_endpoints",
          [](const BuildStrategy &self) { return self.trainers_endpoints_; },
          [](BuildStrategy &self,
             const std::vector<std::string> &trainers_endpoints) {
            self.trainers_endpoints_ = trainers_endpoints;
          })
      .def_property("trainer_id",
                    [](const BuildStrategy &self) { return self.trainer_id_; },
                    [](BuildStrategy &self, int trainer_id) {
                      self.trainer_id_ = trainer_id;
                    })
C
chengduo 已提交
1467 1468 1469 1470 1471 1472
      .def_property(
          "fuse_elewise_add_act_ops",
          [](const BuildStrategy &self) {
            return self.fuse_elewise_add_act_ops_;
          },
          [](BuildStrategy &self, bool b) {
X
Xin Pan 已提交
1473
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
C
chengduo 已提交
1474 1475 1476
            self.fuse_elewise_add_act_ops_ = b;
          },
          R"DOC(The type is BOOL, fuse_elewise_add_act_ops indicate whether
F
flame 已提交
1477 1478 1479 1480 1481 1482 1483 1484 1485 1486
                to fuse elementwise_add_op and activation_op,
                it may make the execution faster. Default False

                Examples:
                    .. code-block:: python

                        import paddle.fluid as fluid
                        build_strategy = fluid.BuildStrategy()
                        build_strategy.fuse_elewise_add_act_ops = True
                     )DOC")
1487 1488 1489 1490 1491 1492 1493 1494 1495 1496
      .def_property(
          "fuse_relu_depthwise_conv",
          [](const BuildStrategy &self) {
            return self.fuse_relu_depthwise_conv_;
          },
          [](BuildStrategy &self, bool b) {
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
            self.fuse_relu_depthwise_conv_ = b;
          },
          R"DOC(The type is BOOL, fuse_relu_depthwise_conv indicate whether
F
flame 已提交
1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508
                to fuse relu and depthwise_conv2d,
                it will save GPU memory and may make the execution faster.
                This options is only available in GPU devices.
                Default False.

                Examples:
                    .. code-block:: python

                        import paddle.fluid as fluid
                        build_strategy = fluid.BuildStrategy()
                        build_strategy.fuse_relu_depthwise_conv = True
          )DOC")
1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521
      .def_property(
          "fuse_broadcast_ops",
          [](const BuildStrategy &self) { return self.fuse_broadcast_ops_; },
          [](BuildStrategy &self, bool b) {
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
            self.fuse_broadcast_ops_ = b;
          },
          R"DOC(The type is BOOL, fuse_broadcast_op indicates whether
                      to fuse the broadcast ops. Note that, in Reduce mode,
                      fusing broadcast ops may make the program faster. Because
                      fusing broadcast OP equals delaying the execution of all
                      broadcast Ops, in this case, all nccl streams are used only
                      for NCCLReduce operations for a period of time. Default False.)DOC")
C
chengduo 已提交
1522 1523 1524 1525 1526 1527 1528 1529 1530
      .def_property("fuse_all_optimizer_ops",
                    [](const BuildStrategy &self) {
                      return self.fuse_all_optimizer_ops_;
                    },
                    [](BuildStrategy &self, bool b) {
                      PADDLE_ENFORCE(!self.IsFinalized(),
                                     "BuildStrategy is finlaized.");
                      self.fuse_all_optimizer_ops_ = b;
                    })
Q
qingqing01 已提交
1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544
      .def_property(
          "sync_batch_norm",
          [](const BuildStrategy &self) { return self.sync_batch_norm_; },
          [](BuildStrategy &self, bool b) {
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
            self.sync_batch_norm_ = b;
          },
          R"DOC(The type is BOOL, sync_batch_norm indicates whether to use
                synchronous batch normalization which synchronizes the mean
                and variance through multi-devices in training phase.

                Current implementation doesn't support FP16 training and CPU.
                And only synchronous on one machine, not all machines.

F
flame 已提交
1545 1546 1547 1548 1549 1550 1551 1552 1553
                Default False

                Examples:
                    .. code-block:: python

                        import paddle.fluid as fluid
                        build_strategy = fluid.BuildStrategy()
                        build_strategy.sync_batch_norm = True
                )DOC")
D
dzhwinter 已提交
1554 1555 1556
      .def_property(
          "memory_optimize",
          [](const BuildStrategy &self) { return self.memory_optimize_; },
1557 1558 1559 1560 1561 1562 1563 1564 1565 1566
          [](BuildStrategy &self, bool b) { self.memory_optimize_ = b; },
          R"DOC(The type is BOOL, memory opitimize aims to save total memory 
                consumption, set to True to enable it.
                
                Memory Optimize is our experimental feature, some variables 
                may be reused/removed by optimize strategy. If you need to
                fetch some variable values when using this feature, please
                set the persistable property of the variables to True.
                
                Default False)DOC")
1567 1568 1569 1570
      .def_property(
          "is_distribution",
          [](const BuildStrategy &self) { return self.is_distribution_; },
          [](BuildStrategy &self, bool b) { self.is_distribution_ = b; })
Q
can run  
Qiao Longfei 已提交
1571 1572 1573
      .def_property("async_mode",
                    [](const BuildStrategy &self) { return self.async_mode_; },
                    [](BuildStrategy &self, bool b) { self.async_mode_ = b; })
D
dzhwinter 已提交
1574
      .def_property(
D
dzhwinter 已提交
1575 1576 1577
          "enable_inplace",
          [](const BuildStrategy &self) { return self.enable_inplace_; },
          [](BuildStrategy &self, bool b) { self.enable_inplace_ = b; })
C
chengduo 已提交
1578 1579 1580 1581
      .def_property(
          "fuse_all_reduce_ops",
          [](const BuildStrategy &self) { return self.fuse_all_reduce_ops_; },
          [](BuildStrategy &self, bool b) { self.fuse_all_reduce_ops_ = b; })
1582 1583 1584 1585
      .def_property(
          "cache_runtime_context",
          [](const BuildStrategy &self) { return self.cache_runtime_context_; },
          [](BuildStrategy &self, bool b) { self.cache_runtime_context_ = b; })
1586 1587 1588 1589 1590 1591 1592 1593 1594
      .def_property(
          "mkldnn_enabled_op_types",
          [](const BuildStrategy &self) {
            return self.mkldnn_enabled_op_types_;
          },
          [](BuildStrategy &self,
             const std::unordered_set<std::string> &mkldnn_enabled_op_types) {
            self.mkldnn_enabled_op_types_ = mkldnn_enabled_op_types;
          })
1595
      .def("_finalize_strategy_and_create_passes",
X
fix  
Xin Pan 已提交
1596
           [](BuildStrategy &self) -> std::shared_ptr<ir::PassBuilder> {
1597 1598 1599 1600 1601
             return self.CreatePassesFromStrategy(true);
           },
           R"DOC(Allow user to customized passes. Normally model-specific
                optimization passes should be defined in this way. BuildStrategy
                cannot be updated after being finalized.)DOC");
Y
yuyang18 已提交
1602 1603

  pe.def(py::init<const std::vector<platform::Place> &,
Y
Yan Xu 已提交
1604
                  const std::vector<std::string> &, const std::string &,
X
Xin Pan 已提交
1605
                  Scope *, std::vector<Scope *> &, const ExecutionStrategy &,
X
Xin Pan 已提交
1606
                  const BuildStrategy &, ir::Graph *>())
Y
Yu Yang 已提交
1607 1608 1609 1610
      // NOTE: even we return a vec<Scope*>* to Python use reference policy.
      // We still cannot get local_scope from this vector, since the element
      // of vec<Scope*> will be freed by Python GC. We can only return Scope*
      // one by one and mark them as reference.
1611 1612 1613 1614 1615
      .def("local_scopes",
           [](ParallelExecutor &self) -> std::vector<Scope *> * {
             return &self.GetLocalScopes();
           },
           py::return_value_policy::reference)
1616 1617 1618
      .def("drop_local_exe_scopes", &ParallelExecutor::DropLocalExeScopes)
      .def("_need_create_local_exe_scopes",
           &ParallelExecutor::NeedCreateLocalExeScope)
Y
Yu Yang 已提交
1619 1620 1621 1622
      .def("feed_tensors_into_local_scopes",
           &ParallelExecutor::FeedTensorsIntoLocalScopes)
      .def("feed_and_split_tensor_into_local_scopes",
           &ParallelExecutor::FeedAndSplitTensorIntoLocalScopes)
S
sneaxiy 已提交
1623 1624 1625 1626 1627 1628
      .def("run", [](ParallelExecutor &self,
                     const std::vector<std::string> &fetch_tensors,
                     const std::string &fetched_var_name) {
        pybind11::gil_scoped_release release;
        self.Run(fetch_tensors, fetched_var_name);
      });
Y
Yu Yang 已提交
1629

1630
  BindRecordIOWriter(&m);
W
Wang Guibao 已提交
1631
  BindAsyncExecutor(&m);
D
dongdaxiang 已提交
1632
  BindFleetWrapper(&m);
W
wopeizl 已提交
1633
#ifndef _WIN32
D
dongdaxiang 已提交
1634
  BindNCCLWrapper(&m);
W
wopeizl 已提交
1635
#endif
F
flame 已提交
1636 1637
  BindGraph(&m);
  BindNode(&m);
F
flame 已提交
1638
  BindInferenceApi(&m);
1639
  BindDataset(&m);
1640 1641 1642
#ifdef PADDLE_WITH_DISTRIBUTE
  BindCommunicator(&m);
#endif
L
Luo Tao 已提交
1643
}
1644
}  // namespace pybind
1645
}  // namespace paddle