pybind.cc 45.5 KB
Newer Older
1
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
2 3 4 5 6

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

7
http://www.apache.org/licenses/LICENSE-2.0
8 9 10 11 12 13

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
L
lgone2000 已提交
14
#include <Python.h>
C
chengduoZH 已提交
15 16
#include <algorithm>
#include <map>
S
sneaxiy 已提交
17
#include <memory>
C
chengduoZH 已提交
18 19 20 21 22
#include <mutex>  // NOLINT // for call_once
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>
23

Y
Yi Wang 已提交
24 25 26
#include "paddle/fluid/framework/executor.h"
#include "paddle/fluid/framework/feed_fetch_method.h"
#include "paddle/fluid/framework/framework.pb.h"
27
#include "paddle/fluid/framework/ir/pass_builder.h"
Y
Yi Wang 已提交
28 29 30
#include "paddle/fluid/framework/lod_rank_table.h"
#include "paddle/fluid/framework/lod_tensor.h"
#include "paddle/fluid/framework/lod_tensor_array.h"
31
#include "paddle/fluid/framework/op_registry.h"
Y
Yu Yang 已提交
32
#include "paddle/fluid/framework/parallel_executor.h"
Y
Yi Wang 已提交
33
#include "paddle/fluid/framework/prune.h"
Y
Refine  
Yu Yang 已提交
34
#include "paddle/fluid/framework/reader.h"
S
sneaxiy 已提交
35
#include "paddle/fluid/framework/scope_pool.h"
Y
Yi Wang 已提交
36
#include "paddle/fluid/framework/selected_rows.h"
X
Xin Pan 已提交
37
#include "paddle/fluid/framework/version.h"
38
#include "paddle/fluid/imperative/layer.h"
Y
Refine  
Yu Yang 已提交
39
#include "paddle/fluid/memory/allocation/allocator_strategy.h"
D
dzhwinter 已提交
40
#include "paddle/fluid/operators/activation_op.h"
S
sneaxiy 已提交
41
#include "paddle/fluid/operators/py_func_op.h"
S
sneaxiy 已提交
42
#include "paddle/fluid/operators/reader/lod_tensor_blocking_queue.h"
Y
Yu Yang 已提交
43
#include "paddle/fluid/platform/cpu_info.h"
Y
Yi Wang 已提交
44
#include "paddle/fluid/platform/enforce.h"
45
#include "paddle/fluid/platform/init.h"
Y
Yi Wang 已提交
46 47
#include "paddle/fluid/platform/place.h"
#include "paddle/fluid/platform/profiler.h"
W
Wang Guibao 已提交
48
#include "paddle/fluid/pybind/async_executor_py.h"
Y
Yi Wang 已提交
49 50
#include "paddle/fluid/pybind/const_value.h"
#include "paddle/fluid/pybind/exception.h"
51
#include "paddle/fluid/pybind/imperative.h"
F
flame 已提交
52
#include "paddle/fluid/pybind/ir.h"
53 54
#include "paddle/fluid/pybind/protobuf.h"
#include "paddle/fluid/pybind/pybind.h"  // NOLINT
Y
Yu Yang 已提交
55
#include "paddle/fluid/pybind/recordio.h"
Y
Yi Wang 已提交
56
#include "paddle/fluid/pybind/tensor_py.h"
Y
Yu Yang 已提交
57

58
#include "paddle/fluid/string/to_string.h"
59

D
Dong Zhihong 已提交
60
#ifdef PADDLE_WITH_CUDA
P
peizhilin 已提交
61
#ifndef _WIN32
Y
Yi Wang 已提交
62
#include "paddle/fluid/operators/nccl/nccl_gpu_common.h"
P
peizhilin 已提交
63
#endif
Y
Yi Wang 已提交
64 65
#include "paddle/fluid/platform/cuda_profiler.h"
#include "paddle/fluid/platform/gpu_info.h"
D
Dong Zhihong 已提交
66 67
#endif

M
minqiyang 已提交
68 69
#include "pybind11/stl.h"

70 71 72 73
DEFINE_bool(reader_queue_speed_test_mode, false,
            "If set true, the queue.pop will only get data from queue but not "
            "remove the data from queue for speed testing");

Q
Qiao Longfei 已提交
74 75 76
// disable auto conversion to list in Python
PYBIND11_MAKE_OPAQUE(paddle::framework::LoDTensorArray);

77
namespace paddle {
78
namespace pybind {
79
bool IsCompiledWithCUDA() {
80
#ifndef PADDLE_WITH_CUDA
Q
qijun 已提交
81 82 83 84 85 86
  return false;
#else
  return true;
#endif
}

87
bool IsCompiledWithBrpc() {
88
#ifndef PADDLE_WITH_DISTRIBUTE
89 90
  return false;
#endif
91 92 93 94 95 96

#ifdef PADDLE_WITH_GRPC
  return false;
#endif

  return true;
97 98
}

Y
update  
Yancey1989 已提交
99
bool IsCompiledWithDIST() {
Y
Yancey1989 已提交
100
#ifdef PADDLE_WITH_DISTRIBUTE
Y
update  
Yancey1989 已提交
101 102 103 104 105 106
  return true;
#else
  return false;
#endif
}

107
PYBIND11_MODULE(core, m) {
Y
Yu Yang 已提交
108 109 110
  // Not used, just make sure cpu_info.cc is linked.
  paddle::platform::CpuTotalPhysicalMemory();

Y
Refine  
Yu Yang 已提交
111
  paddle::memory::allocation::UseAllocatorStrategyGFlag();
112
  m.doc() = "C++ core of PaddlePaddle";
113

114 115 116 117
  // using framework in this function. Since it is inside a function, it will
  // not cause namespace pollution.
  using namespace paddle::framework;  // NOLINT

118
  BindException(&m);
Y
Yu Yang 已提交
119

S
sneaxiy 已提交
120
  m.def(
S
sneaxiy 已提交
121
      "_append_python_callable_object_and_return_id",
S
sneaxiy 已提交
122 123 124 125
      [](py::object py_obj) -> size_t {
        return paddle::operators::AppendPythonCallableObjectAndReturnId(py_obj);
      });

S
sneaxiy 已提交
126 127 128
  m.add_object("_cleanup",
               py::capsule([]() { ScopePool::Instance().Clear(); }));

M
minqiyang 已提交
129
  py::class_<imperative::VarBase>(m, "VarBase", R"DOC()DOC")
130 131
      // .def(py::init<>())
      .def(py::init<bool>(), py::arg("stop_gradient") = false)
132
      .def("_run_backward",
X
Xin Pan 已提交
133
           [](imperative::VarBase &self) { self.RunBackward(); })
M
minqiyang 已提交
134
      .def("_grad_name", &imperative::VarBase::GradName)
M
minqiyang 已提交
135
      .def("_grad_value", &imperative::VarBase::GradValue)
X
Xin Pan 已提交
136
      .def("_clear_gradient", &imperative::VarBase::ClearGradient)
M
minqiyang 已提交
137
      .def("_grad_ivar",
M
minqiyang 已提交
138
           [](const imperative::VarBase &self) { return self.grads_; },
M
minqiyang 已提交
139
           py::return_value_policy::reference)
M
minqiyang 已提交
140
      .def("value", [](const imperative::VarBase &self) { return self.var_; },
M
minqiyang 已提交
141
           py::return_value_policy::reference)
142 143 144 145 146 147
      .def_property(
          "desc",
          [](const imperative::VarBase &self) { return self.var_desc_; },
          [](imperative::VarBase &self, framework::VarDesc *var_desc) {
            self.var_desc_ = var_desc;
          },
148 149 150
          py::return_value_policy::reference)
      .def_property(
          "stop_gradient",
X
Xin Pan 已提交
151
          [](const imperative::VarBase &self) { return self.IsStopGradient(); },
152
          [](imperative::VarBase &self, bool stop_gradient) {
X
Xin Pan 已提交
153
            self.SetStopGradient(stop_gradient);
154
          });
155

156
  py::class_<imperative::OpBase, PyOpBase>(m, "OpBase", R"DOC()DOC")
157 158 159 160 161 162 163 164
      .def(py::init<>())
      .def_property(
          "desc", [](const imperative::OpBase &self) { return self.op_desc_; },
          [](imperative::OpBase &self, framework::OpDesc *op_desc) {
            if (op_desc) {
              self.op_desc_ = op_desc;
            }
          },
X
Xin Pan 已提交
165 166 167 168 169 170 171
          py::return_value_policy::reference)
      .def_property(
          "forward_id",
          [](const imperative::OpBase &self) { return self.forward_id_; },
          [](imperative::OpBase &self, int forward_id) {
            self.forward_id_ = forward_id;
          },
X
Xin Pan 已提交
172 173 174 175 176 177 178
          py::return_value_policy::reference)
      .def_property(
          "backward_id",
          [](const imperative::OpBase &self) { return self.backward_id_; },
          [](imperative::OpBase &self, int backward_id) {
            self.backward_id_ = backward_id;
          },
179 180
          py::return_value_policy::reference);

X
Xin Pan 已提交
181
  py::class_<imperative::Layer, Layer /* <--- trampoline*/> layer(m, "Layer");
182
  layer.def(py::init<>())
X
Xin Pan 已提交
183 184 185
      .def("forward", [](imperative::Layer &self,
                         const std::vector<imperative::VarBase> &inputs) {
        return self.Forward(inputs);
X
Xin Pan 已提交
186
      });
X
Xin Pan 已提交
187

X
polish  
Xin Pan 已提交
188
  py::class_<imperative::PyLayer>(m, "PyLayer")
X
Xin Pan 已提交
189
      .def(py::init<>())
X
Xin Pan 已提交
190 191
      .def_static(
          "apply",
X
Xin Pan 已提交
192
          [](int func_id, const std::vector<imperative::VarBase *> &inputs)
X
Xin Pan 已提交
193 194 195 196
              -> std::vector<imperative::VarBase *> {
                return imperative::PyLayer::Apply(func_id, inputs);
              },
          py::return_value_policy::take_ownership)
X
polish  
Xin Pan 已提交
197 198 199 200 201
      .def_static("register_func",
                  [](int func_id, const py::object &callable) {
                    imperative::PyLayer::RegisterFunc(func_id, callable);
                  })
      .def_static("num_funcs", &imperative::PyLayer::NumFuncs);
X
Xin Pan 已提交
202

203 204
  BindTracer(&m);

205 206 207
  py::class_<Tensor>(m, "Tensor", py::buffer_protocol())
      .def_buffer(
          [](Tensor &self) -> py::buffer_info { return CastToPyBuffer(self); })
Y
yuyang18 已提交
208
      .def("_get_dims",
209
           [](const Tensor &self) { return vectorize(self.dims()); })
Y
yuyang18 已提交
210
      .def("_set_dims",
Q
qijun 已提交
211
           [](Tensor &self, const std::vector<int64_t> &dim) {
Y
Yu Yang 已提交
212
             self.Resize(make_ddim(dim));
Y
Yu Yang 已提交
213
           })
Y
yuyang18 已提交
214
      .def("_set_layout",
D
dzhwinter 已提交
215 216 217
           [](Tensor &self, const std::string &layout) {
             self.set_layout(StringToDataLayout(layout));
           })
Y
yuyang18 已提交
218
      .def("_alloc_float",
D
dzhwinter 已提交
219
           [](Tensor &self, paddle::platform::CUDAPlace &place) {
Q
qijun 已提交
220
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
221
           })
Y
yuyang18 已提交
222
      .def("_alloc_float",
Y
Yu Yang 已提交
223
           [](Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
224
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
225
           })
Y
yuyang18 已提交
226
      .def("_alloc_int",
Y
Yu Yang 已提交
227
           [](Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
228
             self.mutable_data<int>(place);
Y
Yu Yang 已提交
229
           })
Y
yuyang18 已提交
230
      .def("_alloc_int",
D
dzhwinter 已提交
231
           [](Tensor &self, paddle::platform::CUDAPlace &place) {
Q
qijun 已提交
232
             self.mutable_data<int>(place);
Q
qijun 已提交
233
           })
Y
yuyang18 已提交
234
      .def("_alloc_int",
C
chengduoZH 已提交
235 236 237
           [](Tensor &self, paddle::platform::CUDAPinnedPlace &place) {
             self.mutable_data<int>(place);
           })
Y
yuyang18 已提交
238
      .def("_alloc_float",
C
chengduoZH 已提交
239 240 241
           [](Tensor &self, paddle::platform::CUDAPinnedPlace &place) {
             self.mutable_data<float>(place);
           })
Y
Yu Yang 已提交
242 243
      .def("set", PyCPUTensorSetFromArray<float>)
      .def("set", PyCPUTensorSetFromArray<int>)
244
      .def("set", PyCPUTensorSetFromArray<double>)
245
      .def("set", PyCPUTensorSetFromArray<int64_t>)
Y
Yu Yang 已提交
246
      .def("set", PyCPUTensorSetFromArray<bool>)
247
      .def("set", PyCPUTensorSetFromArray<uint16_t>)
F
fengjiayi 已提交
248
      .def("set", PyCPUTensorSetFromArray<uint8_t>)
Q
qingqing01 已提交
249
      .def("set", PyCPUTensorSetFromArray<int8_t>)
250
#ifdef PADDLE_WITH_CUDA
Y
Yu Yang 已提交
251 252
      .def("set", PyCUDATensorSetFromArray<float>)
      .def("set", PyCUDATensorSetFromArray<int>)
253
      .def("set", PyCUDATensorSetFromArray<double>)
254
      .def("set", PyCUDATensorSetFromArray<int64_t>)
Y
Yu Yang 已提交
255
      .def("set", PyCUDATensorSetFromArray<bool>)
256
      .def("set", PyCUDATensorSetFromArray<uint16_t>)
F
fengjiayi 已提交
257
      .def("set", PyCUDATensorSetFromArray<uint8_t>)
Q
qingqing01 已提交
258
      .def("set", PyCUDATensorSetFromArray<int8_t>)
C
chengduoZH 已提交
259 260 261 262 263 264
      .def("set", PyCUDAPinnedTensorSetFromArray<float>)
      .def("set", PyCUDAPinnedTensorSetFromArray<int>)
      .def("set", PyCUDAPinnedTensorSetFromArray<double>)
      .def("set", PyCUDAPinnedTensorSetFromArray<int64_t>)
      .def("set", PyCUDAPinnedTensorSetFromArray<bool>)
      .def("set", PyCUDAPinnedTensorSetFromArray<uint16_t>)
F
fengjiayi 已提交
265
      .def("set", PyCUDAPinnedTensorSetFromArray<uint8_t>)
Q
qingqing01 已提交
266
      .def("set", PyCUDAPinnedTensorSetFromArray<int8_t>)
Q
qijun 已提交
267
#endif
268
      .def("shape", [](Tensor &self) { return vectorize(self.dims()); })
Y
yuyang18 已提交
269 270 271 272
      .def("_set_float_element", TensorSetElement<float>)
      .def("_get_float_element", TensorGetElement<float>)
      .def("_set_double_element", TensorSetElement<double>)
      .def("_get_double_element", TensorGetElement<double>)
Y
Yu Yang 已提交
273
      .def("_dtype", [](Tensor &self) { return self.type(); });
Y
Yu Yang 已提交
274

X
Xin Pan 已提交
275 276 277 278 279 280 281 282 283 284 285 286 287
  py::class_<LoDTensor, Tensor>(m, "LoDTensor", R"DOC(
    LoDTensor is a Tensor with optional LoD information.

    np.array(lod_tensor) can convert LoDTensor to numpy array.
    lod_tensor.lod() can retrieve the LoD information.

    LoD is short for Level of Details and is usually used for varied sequence
    length. You can skip the following comment if you don't need optional LoD.

  For example:
     A LoDTensor X can look like the example below. It contains 2 sequences.
     The first has length 2 and the second has length 3, as described by x.lod.

X
fix doc  
Xin Pan 已提交
288
     The first tensor dimension 5=2+3 is calculated from LoD if it's available.
X
Xin Pan 已提交
289
     It means the total number of sequence element. In X, each element has 2
X
fix doc  
Xin Pan 已提交
290
     columns, hence [5, 2].
X
Xin Pan 已提交
291 292 293

      x.lod  = [[2, 3]]
      x.data = [[1, 2], [3, 4],
X
fix doc  
Xin Pan 已提交
294 295
                [5, 6], [7, 8], [9, 10]]
      x.shape = [5, 2]
X
Xin Pan 已提交
296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318

      LoD can have multiple levels (for example, a paragraph can have multiple
      sentences and a sentence can have multiple words). In the following
      LodTensor Y, the lod_level is 2. It means there are 2 sequence, the
      first sequence length is 2 (has 2 sub-sequences), the second one's
      length is 1. The first sequence's 2 sub-sequences have length 2 and 2,
      respectively. And the second sequence's 1 sub-sequence has length 3.

      y.lod = [[2 1], [2 2 3]]
      y.shape = [2+2+3, ...]

  Note:
      In above description, LoD is length-based. In Paddle internal
      implementation, lod is offset-based. Hence, internally,
      y.lod is represented as [[0, 2, 3], [0, 2, 4, 7]] (length-based
      equivlent would be [[2-0, 3-2], [2-0, 4-2, 7-4]]).

      Sometimes LoD is called recursive_sequence_length to be more
      self-explanatory. In this case, it must be length-based. Due to history
      reasons. when LoD is called lod in public API, it might be offset-based.
      Users should be careful about it.

        )DOC")
319 320
      .def_buffer(
          [](Tensor &self) -> py::buffer_info { return CastToPyBuffer(self); })
321 322 323 324 325 326 327 328 329 330 331 332 333 334
      .def("__init__",
           [](LoDTensor &instance, const std::vector<std::vector<size_t>>
                                       &recursive_sequence_lengths) {
             LoD new_lod;
             new_lod.reserve(recursive_sequence_lengths.size());
             std::copy(recursive_sequence_lengths.begin(),
                       recursive_sequence_lengths.end(),
                       std::back_inserter(new_lod));
             LoD new_offset_lod = ConvertToOffsetBasedLoD(new_lod);
             PADDLE_ENFORCE(
                 CheckLoD(new_offset_lod, -1),
                 "the provided recursive_sequence_lengths info is invalid");
             new (&instance) LoDTensor(new_offset_lod);
           })
Y
Yu Yang 已提交
335
      .def("__init__", [](LoDTensor &instance) { new (&instance) LoDTensor(); })
G
gongweibao 已提交
336 337 338 339 340
      // We implement offset based LOD in C++ while we use length based with
      // Python API. So we changed set_lod to set_recursive_sequence_lengths to
      // avoid misuse.
      // The discussion is here:
      // https://github.com/PaddlePaddle/Paddle/issues/10855
D
dangqingqing 已提交
341
      .def("set_lod",
342
           [](LoDTensor &self, const std::vector<std::vector<size_t>> &lod) {
343
             // the input lod is offset-based level-of-detail info
Y
Yu Yang 已提交
344
             LoD new_lod;
345 346
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
347 348
             PADDLE_ENFORCE(CheckLoD(new_lod, vectorize(self.dims()).front()),
                            "the provided lod info is invalid");
349
             self.set_lod(new_lod);
D
dangqingqing 已提交
350
           })
351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375
      .def("set_recursive_sequence_lengths",
           [](LoDTensor &self, const std::vector<std::vector<size_t>>
                                   &recursive_sequence_lengths) {
             // the input recursive_sequence_lengths is length-based
             // level-of-detail info
             LoD new_lod;
             new_lod.reserve(recursive_sequence_lengths.size());
             std::copy(recursive_sequence_lengths.begin(),
                       recursive_sequence_lengths.end(),
                       std::back_inserter(new_lod));
             LoD new_offset_lod = ConvertToOffsetBasedLoD(new_lod);
             PADDLE_ENFORCE(
                 CheckLoD(new_offset_lod, vectorize(self.dims()).front()),
                 "the provided recursive_sequence_lengths info is invalid");
             self.set_lod(new_offset_lod);
           })
      .def("lod",
           [](LoDTensor &self) -> std::vector<std::vector<size_t>> {
             // output the offset-based lod info
             LoD lod = self.lod();
             std::vector<std::vector<size_t>> new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             return new_lod;
           })
G
gongweibao 已提交
376
      // Set above comments of set_lod.
377 378 379 380 381 382 383 384 385 386 387 388 389
      .def("recursive_sequence_lengths",
           [](LoDTensor &self) -> std::vector<std::vector<size_t>> {
             // output the length-based lod info
             LoD lod = ConvertToLengthBasedLoD(self.lod());
             std::vector<std::vector<size_t>> new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             return new_lod;
           })
      .def("has_valid_recursive_sequence_lengths", [](LoDTensor &self) -> bool {
        // Check that the lod info is valid and match the outermost
        // dimension of the LoDTensor data
        return CheckLoD(self.lod(), vectorize(self.dims()).front());
D
dangqingqing 已提交
390 391
      });

Q
qijun 已提交
392 393 394 395 396 397 398 399 400 401 402
  py::class_<SelectedRows>(m, "SelectedRows")
      .def("__init__",
           [](SelectedRows &instance) { new (&instance) SelectedRows(); })
      .def("__init__",
           [](SelectedRows &instance, const std::vector<int64_t> rows,
              const int64_t &height) {
             new (&instance) SelectedRows(rows, height);
           })
      .def("get_tensor",
           [](SelectedRows &self) { return self.mutable_value(); },
           py::return_value_policy::reference)
403 404
      .def("numel",
           [](SelectedRows &self) -> int64_t { return self.value().numel(); })
Q
qijun 已提交
405 406
      .def("set_height", &SelectedRows::set_height)
      .def("height", &SelectedRows::height)
Q
qijun 已提交
407 408 409 410 411 412 413 414 415
      .def("set_rows",
           [](SelectedRows &self, std::vector<int64_t> rows) {
#ifndef PADDLE_WITH_CUDA
             self.set_rows(rows);
#else
        Vector<int64_t> new_rows(rows);
        self.set_rows(new_rows);
#endif
           })
416
      .def("sync_index", [](SelectedRows &instance) { instance.SyncIndex(); })
417
      .def("rows", [](SelectedRows &self) {
418 419 420 421 422
        auto rows = self.rows();
        std::vector<int64_t> new_rows;
        new_rows.reserve(rows.size());
        std::copy(rows.begin(), rows.end(), std::back_inserter(new_rows));
        return new_rows;
423
      });
Q
qijun 已提交
424

425
  py::class_<Variable>(m, "Variable", R"DOC(Variable Class.
426 427 428

All parameter, weight, gradient are variables in Paddle.
)DOC")
429
      .def("is_int", [](const Variable &var) { return var.IsType<int>(); })
430
      .def("set_int",
431 432
           [](Variable &var, int val) -> void { *var.GetMutable<int>() = val; })
      .def("get_int", [](const Variable &var) -> int { return var.Get<int>(); })
433 434 435 436 437 438 439
      .def("is_float", [](const Variable &var) { return var.IsType<float>(); })
      .def("set_float",
           [](Variable &var, float val) -> void {
             *var.GetMutable<float>() = val;
           })
      .def("get_float",
           [](const Variable &var) -> float { return var.Get<float>(); })
Y
Yu Yang 已提交
440
      .def("get_tensor",
441 442
           [](Variable &self) -> LoDTensor * {
             return self.GetMutable<LoDTensor>();
D
dangqingqing 已提交
443 444
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
445 446 447
      .def("get_lod_rank_table",
           [](Variable &self) { return self.GetMutable<LoDRankTable>(); },
           py::return_value_policy::reference)
Q
qijun 已提交
448 449 450 451 452
      .def("get_selected_rows",
           [](Variable &self) -> SelectedRows * {
             return self.GetMutable<SelectedRows>();
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
453 454 455
      .def("get_lod_tensor_array",
           [](Variable &self) { return self.GetMutable<LoDTensorArray>(); },
           py::return_value_policy::reference)
P
peizhilin 已提交
456
#if (defined(PADDLE_WITH_CUDA) && !defined(_WIN32))
D
Dong Zhihong 已提交
457 458 459 460 461
      .def("get_communicator",
           [](Variable &self) -> platform::Communicator * {
             return self.GetMutable<platform::Communicator>();
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
462
#endif
Y
Refine  
Yu Yang 已提交
463 464 465 466 467
      .def("get_reader",
           [](Variable &self) -> framework::ReaderHolder * {
             PADDLE_ENFORCE(self.IsType<framework::ReaderHolder>());
             return self.GetMutable<framework::ReaderHolder>();
           },
W
wopeizl 已提交
468
           py::return_value_policy::reference);
469

Y
Refine  
Yu Yang 已提交
470
  py::class_<framework::ReaderHolder>(m, "Reader", "")
471
      .def("reset", &framework::ReaderHolder::ResetAll);
Y
Refine  
Yu Yang 已提交
472

S
sneaxiy 已提交
473 474 475 476
  using LoDTensorBlockingQueue =
      ::paddle::operators::reader::LoDTensorBlockingQueue;
  using LoDTensorBlockingQueueHolder =
      ::paddle::operators::reader::LoDTensorBlockingQueueHolder;
S
sneaxiy 已提交
477 478
  py::class_<LoDTensorBlockingQueue, std::shared_ptr<LoDTensorBlockingQueue>>(
      m, "LoDTensorBlockingQueue", "")
S
sneaxiy 已提交
479
      .def("push",
S
sneaxiy 已提交
480
           [](LoDTensorBlockingQueue &self,
S
sneaxiy 已提交
481
              const std::vector<framework::LoDTensor> &lod_tensor_vec) {
S
sneaxiy 已提交
482
             pybind11::gil_scoped_release release;
S
sneaxiy 已提交
483
             return self.Push(lod_tensor_vec);
S
sneaxiy 已提交
484
           })
S
sneaxiy 已提交
485 486 487 488
      .def("size", &LoDTensorBlockingQueue::Size)
      .def("capacity", &LoDTensorBlockingQueue::Cap)
      .def("close", &LoDTensorBlockingQueue::Close)
      .def("is_closed", &LoDTensorBlockingQueue::IsClosed);
S
sneaxiy 已提交
489

S
sneaxiy 已提交
490
  m.def("init_lod_tensor_blocking_queue",
S
sneaxiy 已提交
491
        [](Variable &var, size_t capacity,
S
sneaxiy 已提交
492
           const std::vector<std::vector<int64_t>> &shapes)
S
sneaxiy 已提交
493
            -> std::shared_ptr<LoDTensorBlockingQueue> {
S
sneaxiy 已提交
494 495 496 497 498 499
              std::vector<DDim> dims(shapes.size());
              std::transform(shapes.begin(), shapes.end(), dims.begin(),
                             [](const std::vector<int64_t> &shape) {
                               return make_ddim(shape);
                             });
              auto *holder = var.GetMutable<LoDTensorBlockingQueueHolder>();
500 501
              holder->InitOnce(capacity, dims,
                               FLAGS_reader_queue_speed_test_mode);
S
sneaxiy 已提交
502
              return holder->GetQueue();
S
sneaxiy 已提交
503
            },
S
sneaxiy 已提交
504
        py::return_value_policy::copy);
S
sneaxiy 已提交
505

S
sneaxiy 已提交
506
  py::class_<Scope>(m, "_Scope", R"DOC(
Q
Qiao Longfei 已提交
507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525
    Scope is an association of a name to Variable. All variables belong to Scope.

    Variables in a parent scope can be retrieved from local scope.

    You need to specify a scope to run a Net, i.e., `exe.Run(&scope)`.
    One net can run in different scopes and update different variable in the
    scope.

    You can create var in a scope and get it from the scope.

    Examples:
        .. code-block:: python

          # create tensor from a scope and set value to it.
          param = scope.var('Param').get_tensor()
          param_array = np.full((height, row_numel), 5.0).astype("float32")
          param.set(param_array, place)

        )DOC")
S
sneaxiy 已提交
526 527
      .def("_remove_from_pool",
           [](Scope &self) { ScopePool::Instance().Remove(&self); })
D
dongzhihong 已提交
528
      .def("var",
529
           [](Scope &self, const std::string &name) -> Variable * {
D
dongzhihong 已提交
530
             return self.Var(name);
Y
Yu Yang 已提交
531
           },
532
           py::return_value_policy::reference)
533
      .def("find_var", &Scope::FindVar, py::return_value_policy::reference)
534
      .def("new_scope", [](Scope &self) -> Scope * { return &self.NewScope(); },
535
           py::return_value_policy::reference)
Y
Yu Yang 已提交
536
      .def("drop_kids", &Scope::DropKids);
537

S
sneaxiy 已提交
538 539 540 541 542 543 544 545
  m.def("Scope",
        []() -> Scope * {
          auto *s = new Scope();
          ScopePool::Instance().Insert(std::unique_ptr<Scope>(s));
          return s;
        },
        py::return_value_policy::reference);

Y
Yu Yang 已提交
546 547
  //! @note: Be careful! PyBind will return std::string as an unicode, not
  //! Python str. If you want a str object, you should cast them in Python.
Y
Yu Yang 已提交
548 549
  m.def("get_all_op_protos", []() -> std::vector<py::bytes> {
    std::vector<py::bytes> ret_values;
550 551 552 553 554 555 556 557 558 559
    for (auto &iter : OpInfoMap::Instance().map()) {
      auto &info = iter.second;
      if (info.HasOpProtoAndChecker()) {
        std::string str;
        PADDLE_ENFORCE(
            info.Proto().SerializeToString(&str),
            "Serialize OpProto Error. This could be a bug of Paddle.");
        ret_values.emplace_back(str);
      }
    }
Y
Yu Yang 已提交
560 561
    return ret_values;
  });
562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577
  m.def(
      "get_grad_op_desc", [](const OpDesc &op_desc,
                             const std::unordered_set<std::string> &no_grad_set,
                             const std::vector<BlockDesc *> &grad_sub_block) {
        std::unordered_map<std::string, std::string> grad_to_var;
        std::vector<std::unique_ptr<OpDesc>> grad_op_descs =
            framework::OpInfoMap::Instance()
                .Get(op_desc.Type())
                .GradOpMaker()(op_desc, no_grad_set, &grad_to_var,
                               grad_sub_block);
        std::vector<OpDesc *> grad_op_desc_ptrs(grad_op_descs.size());
        std::transform(grad_op_descs.begin(), grad_op_descs.end(),
                       grad_op_desc_ptrs.begin(),
                       [](std::unique_ptr<OpDesc> &p) { return p.release(); });
        return std::make_pair(grad_op_desc_ptrs, grad_to_var);
      });
Y
Yu Yang 已提交
578
  m.def("prune", [](const ProgramDesc &origin,
579
                    const std::vector<std::array<size_t, 2>> &targets) {
Y
Yu Yang 已提交
580
    ProgramDesc prog_with_targets(origin);
581
    for (const auto &t : targets) {
582
      prog_with_targets.MutableBlock(t[0])->Op(t[1])->SetIsTarget(true);
583
    }
584
    proto::ProgramDesc pruned_desc;
585
    Prune(*prog_with_targets.Proto(), &pruned_desc);
Y
Yu Yang 已提交
586
    return new ProgramDesc(pruned_desc);
587
  });
588 589 590 591
  m.def("empty_var_name",
        []() { return std::string(framework::kEmptyVarName); });
  m.def("grad_var_suffix",
        []() { return std::string(framework::kGradVarSuffix); });
592 593 594
  m.def_submodule(
       "var_names",
       "The module will return special predefined variable name in Paddle")
Y
Yi Wang 已提交
595 596
      .def("empty", []() { return kEmptyVarName; })
      .def("temp", []() { return kTempVarName; });
Q
qijun 已提交
597
  // clang-format off
Y
Yu Yang 已提交
598
  py::class_<paddle::platform::DeviceContext>(m, "DeviceContext")
Q
qijun 已提交
599 600
      .def_static("create",
                  [](paddle::platform::CPUPlace& place)
Q
qijun 已提交
601
                      -> paddle::platform::DeviceContext* {
Q
qijun 已提交
602 603 604
                    return new paddle::platform::CPUDeviceContext();
                  })
      .def_static("create",
D
dzhwinter 已提交
605
                  [](paddle::platform::CUDAPlace& place)
Q
qijun 已提交
606
                      -> paddle::platform::DeviceContext* {
607
#ifndef PADDLE_WITH_CUDA
D
dzhwinter 已提交
608
                    PADDLE_THROW("CUDAPlace is not supported in CPU device.");
Q
qijun 已提交
609
#else
Q
qijun 已提交
610
                    return new paddle::platform::CUDADeviceContext(place);
Q
qijun 已提交
611
#endif
C
chengduoZH 已提交
612 613 614 615 616 617 618 619 620 621 622
                  })
          .def_static("create",
                [](paddle::platform::CUDAPinnedPlace& place)
                        -> paddle::platform::DeviceContext* {
#ifndef PADDLE_WITH_CUDA
                  PADDLE_THROW(
                        "CUDAPinnedPlace is not supported in CPU device.");
#else
                  return new paddle::platform::CUDAPinnedDeviceContext(place);
#endif
                });;
D
Dong Zhihong 已提交
623
// clang-format on
P
peizhilin 已提交
624
#if (defined(PADDLE_WITH_CUDA) && !defined(_WIN32))
D
Dong Zhihong 已提交
625 626
  py::class_<platform::Communicator>(m, "Communicator").def(py::init<>());
#endif
D
dzhwinter 已提交
627
  py::class_<platform::CUDAPlace>(m, "CUDAPlace")
628
      .def(py::init<int>())
D
dzhwinter 已提交
629
      .def("__str__", string::to_string<const platform::CUDAPlace &>);
Q
qijun 已提交
630

631 632 633
  py::class_<paddle::platform::CPUPlace>(m, "CPUPlace")
      .def(py::init<>())
      .def("__str__", string::to_string<const platform::CPUPlace &>);
Y
Yu Yang 已提交
634

C
chengduoZH 已提交
635 636 637 638
  py::class_<paddle::platform::CUDAPinnedPlace>(m, "CUDAPinnedPlace")
      .def(py::init<>())
      .def("__str__", string::to_string<const platform::CUDAPinnedPlace &>);

Y
Yu Yang 已提交
639 640 641 642 643 644 645
  py::class_<platform::Place>(m, "Place")
      .def(py::init<>())
      .def("set_place",
           [](platform::Place &self, const platform::CPUPlace &cpu_place) {
             self = cpu_place;
           })
      .def("set_place",
D
dzhwinter 已提交
646
           [](platform::Place &self, const platform::CUDAPlace &gpu_place) {
Y
Yu Yang 已提交
647
             self = gpu_place;
C
chengduoZH 已提交
648 649
           })
      .def("set_place", [](platform::Place &self,
C
chengduoZH 已提交
650 651
                           const platform::CUDAPinnedPlace &cuda_pinned_place) {
        self = cuda_pinned_place;
C
chengduoZH 已提交
652
      });
Y
Yu Yang 已提交
653

Y
Yu Yang 已提交
654 655 656
  py::class_<OperatorBase>(m, "Operator")
      .def_static("create",
                  [](py::bytes protobin) {
657
                    proto::OpDesc desc;
Y
Yu Yang 已提交
658 659 660 661 662
                    PADDLE_ENFORCE(desc.ParsePartialFromString(protobin),
                                   "Cannot parse user input to OpDesc");
                    PADDLE_ENFORCE(desc.IsInitialized(),
                                   "User OpDesc is not initialized, reason %s",
                                   desc.InitializationErrorString());
663
                    return OpRegistry::CreateOp(desc);
Y
Yu Yang 已提交
664
                  })
665
      .def("run",
666
           [](OperatorBase &self, const Scope &scope,
D
dzhwinter 已提交
667 668 669
              const platform::CPUPlace &place) { self.Run(scope, place); })
      .def("run",
           [](OperatorBase &self, const Scope &scope,
D
dzhwinter 已提交
670
              const platform::CUDAPlace &place) { self.Run(scope, place); })
C
chengduoZH 已提交
671 672 673 674 675
      .def("run",
           [](OperatorBase &self, const Scope &scope,
              const platform::CUDAPinnedPlace &place) {
             self.Run(scope, place);
           })
Y
Yu Yang 已提交
676 677 678 679 680 681 682
      .def("type",
           [](const OperatorBase &op) -> std::string { return op.Type(); })
      .def("outputs",
           [](const OperatorBase &op)
               -> std::map<std::string, std::vector<std::string>> {
                 return op.Outputs();
               })
Q
qijun 已提交
683 684
      .def("output_vars",
           [](const OperatorBase &op) { return op.OutputVars(true); })
Y
Yu Yang 已提交
685
      .def("inputs", [](const OperatorBase &op) { return op.Inputs(); })
Q
qijun 已提交
686
      .def("input_vars", [](const OperatorBase &op) { return op.InputVars(); })
Y
Yu Yang 已提交
687 688 689 690
      .def("__str__", &OperatorBase::DebugString)
      .def("no_intermediate_outputs",
           [](const OperatorBase &op) { return op.OutputVars(false); })
      .def("support_gpu", &OperatorBase::SupportGPU);
Y
Yu Yang 已提交
691

F
fengjiayi 已提交
692
  py::class_<framework::Executor>(m, "Executor")
D
dzhwinter 已提交
693
      .def(py::init<const platform::Place &>())
Y
Yancey1989 已提交
694
      .def("close", &Executor::Close)
S
sneaxiy 已提交
695 696 697 698 699
      .def("run", [](Executor &self, const ProgramDesc &prog, Scope *scope,
                     int block_id, bool create_local_scope, bool create_vars) {
        pybind11::gil_scoped_release release;
        self.Run(prog, scope, block_id, create_local_scope, create_vars);
      });
S
sneaxiy 已提交
700

D
dzhwinter 已提交
701
  m.def("init_gflags", framework::InitGflags);
Y
Yang Yu 已提交
702
  m.def("init_glog", framework::InitGLOG);
X
Xin Pan 已提交
703 704
  m.def("init_devices",
        [](bool init_p2p) { framework::InitDevices(init_p2p); });
705

706
  m.def("is_compiled_with_cuda", IsCompiledWithCUDA);
707
  m.def("is_compiled_with_brpc", IsCompiledWithBrpc);
Y
update  
Yancey1989 已提交
708
  m.def("is_compiled_with_dist", IsCompiledWithDIST);
709 710 711 712 713 714
#ifdef PADDLE_WITH_CUDA
  m.def("is_float16_supported", [](const platform::CUDAPlace &place) -> bool {
    // Only GPUs with Compute Capability >= 53 support float16
    return platform::GetCUDAComputeCapability(place.device) >= 53;
  });
#endif
715

716
  m.def("set_feed_variable", framework::SetFeedVariable);
Q
qijun 已提交
717
  m.def("get_fetch_variable", framework::GetFetchVariable);
718
  m.def("get_variable_tensor", framework::GetVariableTensor);
Q
qijun 已提交
719

X
Xin Pan 已提交
720 721
  m.def("_is_program_version_supported", IsProgramVersionSupported);

722 723 724 725 726
  BindProgramDesc(&m);
  BindBlockDesc(&m);
  BindVarDsec(&m);
  BindOpDesc(&m);
  BindConstValue(&m);
Y
Yu Yang 已提交
727

Y
Yu Yang 已提交
728 729 730 731 732 733 734 735 736
  py::class_<framework::LoDRankTable>(m, "LodRankTable")
      .def("items", [](framework::LoDRankTable &table) {
        std::vector<std::pair<size_t, size_t>> res;
        for (auto &item : table.items()) {
          res.push_back({item.index, item.length});
        }
        return res;
      });

Y
Yu Yang 已提交
737
  py::class_<LoDTensorArray>(m, "LoDTensorArray")
S
sneaxiy 已提交
738 739
      .def("__init__",
           [](LoDTensorArray &instance) { new (&instance) LoDTensorArray(); })
Y
Yu Yang 已提交
740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755
      .def("__getitem__",
           [](LoDTensorArray &self, size_t i) { return &self.at(i); },
           py::return_value_policy::reference)
      .def("__len__", [](LoDTensorArray &self) { return self.size(); })
      .def("__setitem__",
           [](LoDTensorArray &self, size_t i, const LoDTensor &t) {
             PADDLE_ENFORCE_LT(i, self.size());
             self[i].ShareDataWith(t);
             self[i].set_lod(t.lod());
           })
      .def("append", [](LoDTensorArray &self, const LoDTensor &t) {
        self.emplace_back();
        self.back().ShareDataWith(t);
        self.back().set_lod(t.lod());
      });

D
dzhwinter 已提交
756 757 758
  m.def("IsInplace",
        [](std::string op) -> bool { return operators::IsInplace(op); });

Y
Yu Yang 已提交
759
  m.def("op_support_gpu", OpSupportGPU);
D
Dong Zhihong 已提交
760
#ifdef PADDLE_WITH_CUDA
D
Dong Zhihong 已提交
761
  m.def("get_cuda_device_count", platform::GetCUDADeviceCount);
D
dangqingqing 已提交
762

P
peizhilin 已提交
763
#ifndef _WIN32
D
dangqingqing 已提交
764 765 766
  m.def("nvprof_init", platform::CudaProfilerInit);
  m.def("nvprof_start", platform::CudaProfilerStart);
  m.def("nvprof_stop", platform::CudaProfilerStop);
D
Dong Zhihong 已提交
767
#endif
P
peizhilin 已提交
768
#endif
Y
Yu Yang 已提交
769

770 771 772 773
  py::enum_<platform::ProfilerState>(m, "ProfilerState", py::arithmetic())
      .value("kDisabled", platform::ProfilerState::kDisabled)
      .value("kCPU", platform::ProfilerState::kCPU)
      .value("kCUDA", platform::ProfilerState::kCUDA)
774
      .value("kAll", platform::ProfilerState::kAll)
775 776 777 778 779 780 781 782 783 784 785 786 787
      .export_values();

  py::enum_<platform::EventSortingKey>(m, "EventSortingKey", py::arithmetic())
      .value("kDefault", platform::EventSortingKey::kDefault)
      .value("kCalls", platform::EventSortingKey::kCalls)
      .value("kTotal", platform::EventSortingKey::kTotal)
      .value("kMin", platform::EventSortingKey::kMin)
      .value("kMax", platform::EventSortingKey::kMax)
      .value("kAve", platform::EventSortingKey::kAve)
      .export_values();

  m.def("enable_profiler", platform::EnableProfiler);
  m.def("disable_profiler", platform::DisableProfiler);
X
Xin Pan 已提交
788
  m.def("is_profiler_enabled", platform::IsProfileEnabled);
789
  m.def("reset_profiler", platform::ResetProfiler);
Y
Yu Yang 已提交
790

791 792
  py::class_<ir::Pass, std::shared_ptr<ir::Pass>> pass(m, "Pass");
  pass.def(py::init())
793 794 795 796 797
      .def(
          "set_str",
          [](ir::Pass &self, const std::string &name, const std::string &attr) {
            self.Set<std::string>(name, new std::string(attr));
          })
X
Xin Pan 已提交
798 799
      .def("set_int", [](ir::Pass &self, const std::string &name,
                         int val) { self.Set<const int>(name, new int(val)); })
F
flame 已提交
800 801 802 803 804 805
      .def("type", &ir::Pass::Type)
      .def("apply", [](ir::Pass &self, std::shared_ptr<ir::Graph> graph) {
        std::unique_ptr<ir::Graph> origin_graph(graph.get());
        auto optim_graph = self.Apply(std::move(origin_graph));
        graph.reset(optim_graph.release());
      });
806

X
fix  
Xin Pan 已提交
807 808
  py::class_<ir::PassBuilder, std::shared_ptr<ir::PassBuilder>> pb(
      m, "PassBuilder");
809 810 811 812 813 814 815 816 817 818 819 820 821 822
  pb.def(py::init())
      .def("append_pass",
           [](ir::PassBuilder &self,
              const std::string &pass_type) -> std::shared_ptr<ir::Pass> {
             return self.AppendPass(pass_type);
           })
      .def("all_passes", [](ir::PassBuilder &self) { return self.AllPasses(); })
      .def("insert_pass",
           [](ir::PassBuilder &self, size_t idx, const std::string &pass_type) {
             return self.InsertPass(idx, pass_type);
           })
      .def("remove_pass",
           [](ir::PassBuilder &self, size_t idx) { self.RemovePass(idx); });

Y
yuyang18 已提交
823
  // -- python binds for parallel executor.
Y
yuyang18 已提交
824
  py::class_<ParallelExecutor> pe(m, "ParallelExecutor");
C
chengduo 已提交
825 826 827 828
  py::class_<ExecutionStrategy> exec_strategy(pe, "ExecutionStrategy", R"DOC(
    ExecutionStrategy allows the user to more preciously control how to run
    the program in ParallelExecutor by setting the property.

C
chengduo 已提交
829 830 831 832 833 834 835 836 837 838 839
    Examples:
        .. code-block:: python

          exec_strategy = fluid.ExecutionStrategy()
          exec_strategy.num_threads = 4

          train_exe = fluid.ParallelExecutor(use_cuda=True,
                                             loss_name=loss.name,
                                             exec_strategy=exec_strategy)

          train_loss, = train_exe.run([loss.name], feed=feed_dict)
C
chengduo 已提交
840 841 842

        )DOC");

Y
yuyang18 已提交
843
  exec_strategy.def(py::init())
Y
yuyang18 已提交
844 845 846 847 848
      .def_property(
          "num_threads",
          [](const ExecutionStrategy &self) { return self.num_threads_; },
          [](ExecutionStrategy &self, size_t num_threads) {
            self.num_threads_ = num_threads;
C
chengduo 已提交
849 850 851 852 853 854 855 856 857 858
          },
          R"DOC(The type is INT, num_threads represents the size of thread pool that
            used to run the operators of the current program in ParallelExecutor.
            If :math:`num\_threads=1`, all the operators will execute one by one,
            but the order maybe difference between iterations.
            If it is not set, it will be set in ParallelExecutor according to the
            device type and device count, for GPU, :math:`num\_threads=device\_count*4`, for CPU,
            :math:`num\_threads=CPU\_NUM*4`, the explanation of:math:`CPU\_NUM` is in ParallelExecutor.
            if it is not set, ParallelExecutor will get the cpu count by calling
            `multiprocessing.cpu_count()`. Default 0.)DOC")
Y
yuyang18 已提交
859
      .def_property(
860 861 862 863
          "use_cuda",
          [](const ExecutionStrategy &self) { return self.use_cuda_; },
          [](ExecutionStrategy &self, bool use_cuda) {
            self.use_cuda_ = use_cuda;
C
chengduo 已提交
864 865 866 867
          })  // FIXME(chengduo): Doesn't add doc for 'use_cuda', use_cuda may
      // make user confuse, because ParallelExecutor has a parameter named
      // 'use_cuda' too, in current implementation, ParallelExecutor's
      // 'use_cuda' will rewrite ExecutionStrategy's 'use_cuda'.
Y
yuyang18 已提交
868 869 870 871 872
      .def_property(
          "allow_op_delay",
          [](const ExecutionStrategy &self) { return self.allow_op_delay_; },
          [](ExecutionStrategy &self, bool allow_op_delay) {
            self.allow_op_delay_ = allow_op_delay;
C
chengduo 已提交
873 874 875 876
          },
          R"DOC(The type is BOOL, allow_op_delay represents whether to delay the
                communication operators to run, it may make the execution faster.
                Note that in some models, allow_op_delay may cause program hang. Default False.)DOC")
Y
yuyang18 已提交
877 878 879 880 881 882 883
      .def_property(
          "num_iteration_per_drop_scope",
          [](const ExecutionStrategy &self) {
            return self.num_iteration_per_drop_scope_;
          },
          [](ExecutionStrategy &self, size_t num_iteration_per_drop_scope) {
            self.num_iteration_per_drop_scope_ = num_iteration_per_drop_scope;
C
chengduo 已提交
884 885 886 887 888 889 890 891 892 893 894
          },
          R"DOC(The type is INT, num_iteration_per_drop_scope indicates how
                many iterations to clean up the temp variables which
                is generated during execution. It may make the execution faster,
                because the temp variable's shape maybe the same between two iterations. Default 100.

                NOTES:
                    1. If you fetch data when calling the 'run', the ParallelExecutor
                       will clean up the temp variables at the end of the current iteration.
                    2. In some NLP model, it may cause the GPU memory is insufficient,
                       in this case, you should reduce `num_iteration_per_drop_scope`.
895 896 897 898 899 900
              )DOC")
      .def_property("_dry_run",
                    [](const ExecutionStrategy &self) { return self.dry_run_; },
                    [](ExecutionStrategy &self, bool dry_run) {
                      self.dry_run_ = dry_run;
                    });
C
chengduo 已提交
901

Y
yuyang18 已提交
902
  exec_strategy.def_property(
Y
yuyang18 已提交
903 904 905 906 907 908 909
      "use_experimental_executor",
      [](const ExecutionStrategy &self) {
        return self.type_ == ExecutionStrategy::kExperimental;
      },
      [](ExecutionStrategy &self, bool experimental) {
        self.type_ = experimental ? ExecutionStrategy::kExperimental
                                  : ExecutionStrategy::kDefault;
Y
yuyang18 已提交
910 911
      });

C
chengduo 已提交
912 913 914 915
  py::class_<BuildStrategy> build_strategy(pe, "BuildStrategy", R"DOC(
    BuildStrategy allows the user to more preciously control how to
    build the SSA Graph in ParallelExecutor by setting the property.

C
chengduo 已提交
916 917 918 919 920 921 922 923 924 925 926
    Examples:
        .. code-block:: python

          build_strategy = fluid.BuildStrategy()
          build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce

          train_exe = fluid.ParallelExecutor(use_cuda=True,
                                             loss_name=loss.name,
                                             build_strategy=build_strategy)

          train_loss, = train_exe.run([loss.name], feed=feed_dict)
C
chengduo 已提交
927
)DOC");
Y
yuyang18 已提交
928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943

  py::enum_<BuildStrategy::ReduceStrategy>(build_strategy, "ReduceStrategy")
      .value("Reduce", BuildStrategy::ReduceStrategy::kReduce)
      .value("AllReduce", BuildStrategy::ReduceStrategy::kAllReduce);
  py::enum_<BuildStrategy::GradientScaleStrategy>(build_strategy,
                                                  "GradientScaleStrategy")
      .value("CoeffNumDevice",
             BuildStrategy::GradientScaleStrategy::kCoeffNumDevice)
      .value("One", BuildStrategy::GradientScaleStrategy::kOne)
      .value("Customized", BuildStrategy::GradientScaleStrategy::kCustomized);

  build_strategy.def(py::init())
      .def_property(
          "reduce_strategy",
          [](const BuildStrategy &self) { return self.reduce_; },
          [](BuildStrategy &self, BuildStrategy::ReduceStrategy strategy) {
X
Xin Pan 已提交
944
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
Y
yuyang18 已提交
945
            self.reduce_ = strategy;
C
chengduo 已提交
946 947 948 949 950 951 952
          },
          R"DOC(The type is STR, there are two reduce strategies in ParallelExecutor,
                  'AllReduce' and 'Reduce'. If you want that all the parameters'
                  optimization are done on all devices independently, you should choose 'AllReduce';
                  if you choose 'Reduce', all the parameters' optimization will be evenly distributed
                  to different devices, and then broadcast the optimized parameter to other devices.
                  In some models, `Reduce` is faster. Default 'AllReduce'. )DOC")
Y
yuyang18 已提交
953 954 955 956 957
      .def_property(
          "gradient_scale_strategy",
          [](const BuildStrategy &self) { return self.gradient_scale_; },
          [](BuildStrategy &self,
             BuildStrategy::GradientScaleStrategy strategy) {
X
Xin Pan 已提交
958
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
Y
yuyang18 已提交
959
            self.gradient_scale_ = strategy;
C
chengduo 已提交
960 961 962 963 964 965
          },
          R"DOC(The type is STR, there are three ways of defining :math:`loss@grad` in
                   ParallelExecutor, 'CoeffNumDevice', 'One' and 'Customized'. By default,
                   ParallelExecutor sets the :math:`loss@grad` according to the number of devices.
                   If you want to customize :math:`loss@grad`, you can choose 'Customized'.
                   Default 'CoeffNumDevice'.)DOC")
Y
yuyang18 已提交
966 967 968 969
      .def_property(
          "debug_graphviz_path",
          [](const BuildStrategy &self) { return self.debug_graphviz_path_; },
          [](BuildStrategy &self, const std::string &path) {
X
Xin Pan 已提交
970
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
Y
yuyang18 已提交
971
            self.debug_graphviz_path_ = path;
C
chengduo 已提交
972 973 974 975
          },
          R"DOC(The type is STR, debug_graphviz_path indicate the path that
                    writing the SSA Graph to file in the form of graphviz, you.
                    It is useful for debugging. Default "")DOC")
S
sneaxiy 已提交
976 977 978 979 980 981
      .def_property(
          "enable_sequential_execution",
          [](const BuildStrategy &self) {
            return self.enable_sequential_execution_;
          },
          [](BuildStrategy &self, bool b) {
X
Xin Pan 已提交
982
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
S
sneaxiy 已提交
983 984 985 986 987 988 989 990 991
            self.enable_sequential_execution_ = b;
          },
          R"DOC(The type is BOOL. If set True, the execution order of ops would be the same as what is in the program. Default False.)DOC")
      .def_property(
          "remove_unnecessary_lock",
          [](const BuildStrategy &self) {
            return self.remove_unnecessary_lock_;
          },
          [](BuildStrategy &self, bool b) {
X
Xin Pan 已提交
992
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
S
sneaxiy 已提交
993 994 995
            self.remove_unnecessary_lock_ = b;
          },
          R"DOC(The type is BOOL. If set True, some locks in GPU ops would be released and ParallelExecutor would run faster. Default False.)DOC")
996 997 998 999 1000 1001
      .def_property(
          "num_trainers",
          [](const BuildStrategy &self) { return self.num_trainers_; },
          [](BuildStrategy &self, int num_trainers) {
            self.num_trainers_ = num_trainers;
          })
1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013
      .def_property(
          "trainers_endpoints",
          [](const BuildStrategy &self) { return self.trainers_endpoints_; },
          [](BuildStrategy &self,
             const std::vector<std::string> &trainers_endpoints) {
            self.trainers_endpoints_ = trainers_endpoints;
          })
      .def_property("trainer_id",
                    [](const BuildStrategy &self) { return self.trainer_id_; },
                    [](BuildStrategy &self, int trainer_id) {
                      self.trainer_id_ = trainer_id;
                    })
C
chengduo 已提交
1014 1015 1016 1017 1018 1019
      .def_property(
          "fuse_elewise_add_act_ops",
          [](const BuildStrategy &self) {
            return self.fuse_elewise_add_act_ops_;
          },
          [](BuildStrategy &self, bool b) {
X
Xin Pan 已提交
1020
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
C
chengduo 已提交
1021 1022 1023 1024 1025
            self.fuse_elewise_add_act_ops_ = b;
          },
          R"DOC(The type is BOOL, fuse_elewise_add_act_ops indicate whether
                     to fuse elementwise_add_op and activation_op,
                     it may make the execution faster. Default False)DOC")
D
dzhwinter 已提交
1026 1027 1028 1029
      .def_property(
          "memory_optimize",
          [](const BuildStrategy &self) { return self.memory_optimize_; },
          [](BuildStrategy &self, bool b) { self.memory_optimize_ = b; })
1030 1031 1032 1033
      .def_property(
          "is_distribution",
          [](const BuildStrategy &self) { return self.is_distribution_; },
          [](BuildStrategy &self, bool b) { self.is_distribution_ = b; })
D
dzhwinter 已提交
1034 1035 1036 1037
      .def_property(
          "memory_early_delete",
          [](const BuildStrategy &self) { return self.memory_early_delete_; },
          [](BuildStrategy &self, bool b) { self.memory_early_delete_ = b; })
1038
      .def("_finalize_strategy_and_create_passes",
X
fix  
Xin Pan 已提交
1039
           [](BuildStrategy &self) -> std::shared_ptr<ir::PassBuilder> {
1040 1041 1042 1043 1044
             return self.CreatePassesFromStrategy(true);
           },
           R"DOC(Allow user to customized passes. Normally model-specific
                optimization passes should be defined in this way. BuildStrategy
                cannot be updated after being finalized.)DOC");
Y
yuyang18 已提交
1045 1046 1047

  pe.def(py::init<const std::vector<platform::Place> &,
                  const std::unordered_set<std::string> &, const ProgramDesc &,
Y
yuyang18 已提交
1048
                  const std::string &, Scope *, std::vector<Scope *> &,
1049
                  const ExecutionStrategy &, const BuildStrategy &>())
Y
Yu Yang 已提交
1050 1051 1052 1053
      // NOTE: even we return a vec<Scope*>* to Python use reference policy.
      // We still cannot get local_scope from this vector, since the element
      // of vec<Scope*> will be freed by Python GC. We can only return Scope*
      // one by one and mark them as reference.
1054 1055 1056 1057 1058
      .def("local_scopes",
           [](ParallelExecutor &self) -> std::vector<Scope *> * {
             return &self.GetLocalScopes();
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
1059 1060 1061 1062
      .def("feed_tensors_into_local_scopes",
           &ParallelExecutor::FeedTensorsIntoLocalScopes)
      .def("feed_and_split_tensor_into_local_scopes",
           &ParallelExecutor::FeedAndSplitTensorIntoLocalScopes)
S
sneaxiy 已提交
1063 1064 1065 1066 1067 1068
      .def("run", [](ParallelExecutor &self,
                     const std::vector<std::string> &fetch_tensors,
                     const std::string &fetched_var_name) {
        pybind11::gil_scoped_release release;
        self.Run(fetch_tensors, fetched_var_name);
      });
Y
Yu Yang 已提交
1069

1070
  BindRecordIOWriter(&m);
W
Wang Guibao 已提交
1071
  BindAsyncExecutor(&m);
F
flame 已提交
1072 1073 1074

  BindGraph(&m);
  BindNode(&m);
L
Luo Tao 已提交
1075
}
1076
}  // namespace pybind
1077
}  // namespace paddle