pybind.cc 46.2 KB
Newer Older
1
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
2 3 4 5 6

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

7
http://www.apache.org/licenses/LICENSE-2.0
8 9 10 11 12 13

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
L
lgone2000 已提交
14
#include <Python.h>
C
chengduoZH 已提交
15 16
#include <algorithm>
#include <map>
S
sneaxiy 已提交
17
#include <memory>
C
chengduoZH 已提交
18 19 20 21 22
#include <mutex>  // NOLINT // for call_once
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>
23

Y
Yi Wang 已提交
24 25 26
#include "paddle/fluid/framework/executor.h"
#include "paddle/fluid/framework/feed_fetch_method.h"
#include "paddle/fluid/framework/framework.pb.h"
27
#include "paddle/fluid/framework/ir/pass_builder.h"
Y
Yi Wang 已提交
28 29 30
#include "paddle/fluid/framework/lod_rank_table.h"
#include "paddle/fluid/framework/lod_tensor.h"
#include "paddle/fluid/framework/lod_tensor_array.h"
31
#include "paddle/fluid/framework/op_registry.h"
Y
Yu Yang 已提交
32
#include "paddle/fluid/framework/parallel_executor.h"
Y
Yi Wang 已提交
33
#include "paddle/fluid/framework/prune.h"
Y
Refine  
Yu Yang 已提交
34
#include "paddle/fluid/framework/reader.h"
S
sneaxiy 已提交
35
#include "paddle/fluid/framework/scope_pool.h"
Y
Yi Wang 已提交
36
#include "paddle/fluid/framework/selected_rows.h"
X
Xin Pan 已提交
37
#include "paddle/fluid/framework/version.h"
38
#include "paddle/fluid/imperative/layer.h"
Y
Refine  
Yu Yang 已提交
39
#include "paddle/fluid/memory/allocation/allocator_strategy.h"
D
dzhwinter 已提交
40
#include "paddle/fluid/operators/activation_op.h"
S
sneaxiy 已提交
41
#include "paddle/fluid/operators/py_func_op.h"
S
sneaxiy 已提交
42
#include "paddle/fluid/operators/reader/lod_tensor_blocking_queue.h"
Y
Yu Yang 已提交
43
#include "paddle/fluid/platform/cpu_info.h"
Y
Yi Wang 已提交
44
#include "paddle/fluid/platform/enforce.h"
45
#include "paddle/fluid/platform/init.h"
Y
Yi Wang 已提交
46 47
#include "paddle/fluid/platform/place.h"
#include "paddle/fluid/platform/profiler.h"
W
Wang Guibao 已提交
48
#include "paddle/fluid/pybind/async_executor_py.h"
Y
Yi Wang 已提交
49 50
#include "paddle/fluid/pybind/const_value.h"
#include "paddle/fluid/pybind/exception.h"
51
#include "paddle/fluid/pybind/imperative.h"
F
flame 已提交
52
#include "paddle/fluid/pybind/inference_api.h"
F
flame 已提交
53
#include "paddle/fluid/pybind/ir.h"
54 55
#include "paddle/fluid/pybind/protobuf.h"
#include "paddle/fluid/pybind/pybind.h"  // NOLINT
Y
Yu Yang 已提交
56
#include "paddle/fluid/pybind/recordio.h"
Y
Yi Wang 已提交
57
#include "paddle/fluid/pybind/tensor_py.h"
Y
Yu Yang 已提交
58

59
#include "paddle/fluid/string/to_string.h"
60

D
Dong Zhihong 已提交
61
#ifdef PADDLE_WITH_CUDA
P
peizhilin 已提交
62
#ifndef _WIN32
Y
Yi Wang 已提交
63
#include "paddle/fluid/operators/nccl/nccl_gpu_common.h"
P
peizhilin 已提交
64
#endif
Y
Yi Wang 已提交
65 66
#include "paddle/fluid/platform/cuda_profiler.h"
#include "paddle/fluid/platform/gpu_info.h"
D
Dong Zhihong 已提交
67 68
#endif

M
minqiyang 已提交
69 70
#include "pybind11/stl.h"

71 72 73 74
DEFINE_bool(reader_queue_speed_test_mode, false,
            "If set true, the queue.pop will only get data from queue but not "
            "remove the data from queue for speed testing");

Q
Qiao Longfei 已提交
75 76 77
// disable auto conversion to list in Python
PYBIND11_MAKE_OPAQUE(paddle::framework::LoDTensorArray);

78
namespace paddle {
79
namespace pybind {
80
bool IsCompiledWithCUDA() {
81
#ifndef PADDLE_WITH_CUDA
Q
qijun 已提交
82 83 84 85 86 87
  return false;
#else
  return true;
#endif
}

88
bool IsCompiledWithBrpc() {
89
#ifndef PADDLE_WITH_DISTRIBUTE
90 91
  return false;
#endif
92 93 94 95 96 97

#ifdef PADDLE_WITH_GRPC
  return false;
#endif

  return true;
98 99
}

Y
update  
Yancey1989 已提交
100
bool IsCompiledWithDIST() {
Y
Yancey1989 已提交
101
#ifdef PADDLE_WITH_DISTRIBUTE
Y
update  
Yancey1989 已提交
102 103 104 105 106 107
  return true;
#else
  return false;
#endif
}

108
PYBIND11_MODULE(core, m) {
Y
Yu Yang 已提交
109 110 111
  // Not used, just make sure cpu_info.cc is linked.
  paddle::platform::CpuTotalPhysicalMemory();

Y
Refine  
Yu Yang 已提交
112
  paddle::memory::allocation::UseAllocatorStrategyGFlag();
113
  m.doc() = "C++ core of PaddlePaddle";
114

115 116 117 118
  // using framework in this function. Since it is inside a function, it will
  // not cause namespace pollution.
  using namespace paddle::framework;  // NOLINT

119
  BindException(&m);
Y
Yu Yang 已提交
120

S
sneaxiy 已提交
121
  m.def(
S
sneaxiy 已提交
122
      "_append_python_callable_object_and_return_id",
S
sneaxiy 已提交
123 124 125 126
      [](py::object py_obj) -> size_t {
        return paddle::operators::AppendPythonCallableObjectAndReturnId(py_obj);
      });

S
sneaxiy 已提交
127 128 129
  m.add_object("_cleanup",
               py::capsule([]() { ScopePool::Instance().Clear(); }));

M
minqiyang 已提交
130
  py::class_<imperative::VarBase>(m, "VarBase", R"DOC()DOC")
131 132
      // .def(py::init<>())
      .def(py::init<bool>(), py::arg("stop_gradient") = false)
133
      .def("_run_backward",
X
Xin Pan 已提交
134
           [](imperative::VarBase &self) { self.RunBackward(); })
M
minqiyang 已提交
135
      .def("_grad_name", &imperative::VarBase::GradName)
M
minqiyang 已提交
136
      .def("_grad_value", &imperative::VarBase::GradValue)
X
Xin Pan 已提交
137
      .def("_clear_gradient", &imperative::VarBase::ClearGradient)
M
minqiyang 已提交
138
      .def("_grad_ivar",
M
minqiyang 已提交
139
           [](const imperative::VarBase &self) { return self.grads_; },
M
minqiyang 已提交
140
           py::return_value_policy::reference)
M
minqiyang 已提交
141
      .def("value", [](const imperative::VarBase &self) { return self.var_; },
M
minqiyang 已提交
142
           py::return_value_policy::reference)
143 144 145 146 147 148
      .def_property(
          "desc",
          [](const imperative::VarBase &self) { return self.var_desc_; },
          [](imperative::VarBase &self, framework::VarDesc *var_desc) {
            self.var_desc_ = var_desc;
          },
149 150 151
          py::return_value_policy::reference)
      .def_property(
          "stop_gradient",
X
Xin Pan 已提交
152
          [](const imperative::VarBase &self) { return self.IsStopGradient(); },
153
          [](imperative::VarBase &self, bool stop_gradient) {
X
Xin Pan 已提交
154
            self.SetStopGradient(stop_gradient);
155
          });
156

157
  py::class_<imperative::OpBase, PyOpBase>(m, "OpBase", R"DOC()DOC")
158 159 160 161 162 163 164 165
      .def(py::init<>())
      .def_property(
          "desc", [](const imperative::OpBase &self) { return self.op_desc_; },
          [](imperative::OpBase &self, framework::OpDesc *op_desc) {
            if (op_desc) {
              self.op_desc_ = op_desc;
            }
          },
X
Xin Pan 已提交
166 167 168 169 170 171 172
          py::return_value_policy::reference)
      .def_property(
          "forward_id",
          [](const imperative::OpBase &self) { return self.forward_id_; },
          [](imperative::OpBase &self, int forward_id) {
            self.forward_id_ = forward_id;
          },
X
Xin Pan 已提交
173 174 175 176 177 178 179
          py::return_value_policy::reference)
      .def_property(
          "backward_id",
          [](const imperative::OpBase &self) { return self.backward_id_; },
          [](imperative::OpBase &self, int backward_id) {
            self.backward_id_ = backward_id;
          },
180 181
          py::return_value_policy::reference);

X
Xin Pan 已提交
182
  py::class_<imperative::Layer, Layer /* <--- trampoline*/> layer(m, "Layer");
183
  layer.def(py::init<>())
X
Xin Pan 已提交
184 185 186
      .def("forward", [](imperative::Layer &self,
                         const std::vector<imperative::VarBase> &inputs) {
        return self.Forward(inputs);
X
Xin Pan 已提交
187
      });
X
Xin Pan 已提交
188

X
polish  
Xin Pan 已提交
189
  py::class_<imperative::PyLayer>(m, "PyLayer")
X
Xin Pan 已提交
190
      .def(py::init<>())
X
Xin Pan 已提交
191 192
      .def_static(
          "apply",
X
Xin Pan 已提交
193
          [](int func_id, const std::vector<imperative::VarBase *> &inputs)
X
Xin Pan 已提交
194 195 196 197
              -> std::vector<imperative::VarBase *> {
                return imperative::PyLayer::Apply(func_id, inputs);
              },
          py::return_value_policy::take_ownership)
X
polish  
Xin Pan 已提交
198 199 200 201 202
      .def_static("register_func",
                  [](int func_id, const py::object &callable) {
                    imperative::PyLayer::RegisterFunc(func_id, callable);
                  })
      .def_static("num_funcs", &imperative::PyLayer::NumFuncs);
X
Xin Pan 已提交
203

204 205
  BindTracer(&m);

206 207 208
  py::class_<Tensor>(m, "Tensor", py::buffer_protocol())
      .def_buffer(
          [](Tensor &self) -> py::buffer_info { return CastToPyBuffer(self); })
Y
yuyang18 已提交
209
      .def("_get_dims",
210
           [](const Tensor &self) { return vectorize(self.dims()); })
Y
yuyang18 已提交
211
      .def("_set_dims",
Q
qijun 已提交
212
           [](Tensor &self, const std::vector<int64_t> &dim) {
Y
Yu Yang 已提交
213
             self.Resize(make_ddim(dim));
Y
Yu Yang 已提交
214
           })
Y
yuyang18 已提交
215
      .def("_set_layout",
D
dzhwinter 已提交
216 217 218
           [](Tensor &self, const std::string &layout) {
             self.set_layout(StringToDataLayout(layout));
           })
Y
yuyang18 已提交
219
      .def("_alloc_float",
D
dzhwinter 已提交
220
           [](Tensor &self, paddle::platform::CUDAPlace &place) {
Q
qijun 已提交
221
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
222
           })
Y
yuyang18 已提交
223
      .def("_alloc_float",
Y
Yu Yang 已提交
224
           [](Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
225
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
226
           })
Y
yuyang18 已提交
227
      .def("_alloc_int",
Y
Yu Yang 已提交
228
           [](Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
229
             self.mutable_data<int>(place);
Y
Yu Yang 已提交
230
           })
Y
yuyang18 已提交
231
      .def("_alloc_int",
D
dzhwinter 已提交
232
           [](Tensor &self, paddle::platform::CUDAPlace &place) {
Q
qijun 已提交
233
             self.mutable_data<int>(place);
Q
qijun 已提交
234
           })
Y
yuyang18 已提交
235
      .def("_alloc_int",
C
chengduoZH 已提交
236 237 238
           [](Tensor &self, paddle::platform::CUDAPinnedPlace &place) {
             self.mutable_data<int>(place);
           })
Y
yuyang18 已提交
239
      .def("_alloc_float",
C
chengduoZH 已提交
240 241 242
           [](Tensor &self, paddle::platform::CUDAPinnedPlace &place) {
             self.mutable_data<float>(place);
           })
Y
Yu Yang 已提交
243 244
      .def("set", PyCPUTensorSetFromArray<float>)
      .def("set", PyCPUTensorSetFromArray<int>)
245
      .def("set", PyCPUTensorSetFromArray<double>)
246
      .def("set", PyCPUTensorSetFromArray<int64_t>)
Y
Yu Yang 已提交
247
      .def("set", PyCPUTensorSetFromArray<bool>)
248
      .def("set", PyCPUTensorSetFromArray<uint16_t>)
F
fengjiayi 已提交
249
      .def("set", PyCPUTensorSetFromArray<uint8_t>)
Q
qingqing01 已提交
250
      .def("set", PyCPUTensorSetFromArray<int8_t>)
251
#ifdef PADDLE_WITH_CUDA
Y
Yu Yang 已提交
252 253
      .def("set", PyCUDATensorSetFromArray<float>)
      .def("set", PyCUDATensorSetFromArray<int>)
254
      .def("set", PyCUDATensorSetFromArray<double>)
255
      .def("set", PyCUDATensorSetFromArray<int64_t>)
Y
Yu Yang 已提交
256
      .def("set", PyCUDATensorSetFromArray<bool>)
257
      .def("set", PyCUDATensorSetFromArray<uint16_t>)
F
fengjiayi 已提交
258
      .def("set", PyCUDATensorSetFromArray<uint8_t>)
Q
qingqing01 已提交
259
      .def("set", PyCUDATensorSetFromArray<int8_t>)
C
chengduoZH 已提交
260 261 262 263 264 265
      .def("set", PyCUDAPinnedTensorSetFromArray<float>)
      .def("set", PyCUDAPinnedTensorSetFromArray<int>)
      .def("set", PyCUDAPinnedTensorSetFromArray<double>)
      .def("set", PyCUDAPinnedTensorSetFromArray<int64_t>)
      .def("set", PyCUDAPinnedTensorSetFromArray<bool>)
      .def("set", PyCUDAPinnedTensorSetFromArray<uint16_t>)
F
fengjiayi 已提交
266
      .def("set", PyCUDAPinnedTensorSetFromArray<uint8_t>)
Q
qingqing01 已提交
267
      .def("set", PyCUDAPinnedTensorSetFromArray<int8_t>)
Q
qijun 已提交
268
#endif
269
      .def("shape", [](Tensor &self) { return vectorize(self.dims()); })
Y
yuyang18 已提交
270 271 272 273
      .def("_set_float_element", TensorSetElement<float>)
      .def("_get_float_element", TensorGetElement<float>)
      .def("_set_double_element", TensorSetElement<double>)
      .def("_get_double_element", TensorGetElement<double>)
Y
Yu Yang 已提交
274
      .def("_dtype", [](Tensor &self) { return self.type(); });
Y
Yu Yang 已提交
275

X
Xin Pan 已提交
276 277 278 279 280 281 282 283 284 285 286 287 288
  py::class_<LoDTensor, Tensor>(m, "LoDTensor", R"DOC(
    LoDTensor is a Tensor with optional LoD information.

    np.array(lod_tensor) can convert LoDTensor to numpy array.
    lod_tensor.lod() can retrieve the LoD information.

    LoD is short for Level of Details and is usually used for varied sequence
    length. You can skip the following comment if you don't need optional LoD.

  For example:
     A LoDTensor X can look like the example below. It contains 2 sequences.
     The first has length 2 and the second has length 3, as described by x.lod.

X
fix doc  
Xin Pan 已提交
289
     The first tensor dimension 5=2+3 is calculated from LoD if it's available.
X
Xin Pan 已提交
290
     It means the total number of sequence element. In X, each element has 2
X
fix doc  
Xin Pan 已提交
291
     columns, hence [5, 2].
X
Xin Pan 已提交
292 293 294

      x.lod  = [[2, 3]]
      x.data = [[1, 2], [3, 4],
X
fix doc  
Xin Pan 已提交
295 296
                [5, 6], [7, 8], [9, 10]]
      x.shape = [5, 2]
X
Xin Pan 已提交
297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319

      LoD can have multiple levels (for example, a paragraph can have multiple
      sentences and a sentence can have multiple words). In the following
      LodTensor Y, the lod_level is 2. It means there are 2 sequence, the
      first sequence length is 2 (has 2 sub-sequences), the second one's
      length is 1. The first sequence's 2 sub-sequences have length 2 and 2,
      respectively. And the second sequence's 1 sub-sequence has length 3.

      y.lod = [[2 1], [2 2 3]]
      y.shape = [2+2+3, ...]

  Note:
      In above description, LoD is length-based. In Paddle internal
      implementation, lod is offset-based. Hence, internally,
      y.lod is represented as [[0, 2, 3], [0, 2, 4, 7]] (length-based
      equivlent would be [[2-0, 3-2], [2-0, 4-2, 7-4]]).

      Sometimes LoD is called recursive_sequence_length to be more
      self-explanatory. In this case, it must be length-based. Due to history
      reasons. when LoD is called lod in public API, it might be offset-based.
      Users should be careful about it.

        )DOC")
320 321
      .def_buffer(
          [](Tensor &self) -> py::buffer_info { return CastToPyBuffer(self); })
322 323 324 325 326 327 328 329 330 331 332 333 334 335
      .def("__init__",
           [](LoDTensor &instance, const std::vector<std::vector<size_t>>
                                       &recursive_sequence_lengths) {
             LoD new_lod;
             new_lod.reserve(recursive_sequence_lengths.size());
             std::copy(recursive_sequence_lengths.begin(),
                       recursive_sequence_lengths.end(),
                       std::back_inserter(new_lod));
             LoD new_offset_lod = ConvertToOffsetBasedLoD(new_lod);
             PADDLE_ENFORCE(
                 CheckLoD(new_offset_lod, -1),
                 "the provided recursive_sequence_lengths info is invalid");
             new (&instance) LoDTensor(new_offset_lod);
           })
Y
Yu Yang 已提交
336
      .def("__init__", [](LoDTensor &instance) { new (&instance) LoDTensor(); })
G
gongweibao 已提交
337 338 339 340 341
      // We implement offset based LOD in C++ while we use length based with
      // Python API. So we changed set_lod to set_recursive_sequence_lengths to
      // avoid misuse.
      // The discussion is here:
      // https://github.com/PaddlePaddle/Paddle/issues/10855
D
dangqingqing 已提交
342
      .def("set_lod",
343
           [](LoDTensor &self, const std::vector<std::vector<size_t>> &lod) {
344
             // the input lod is offset-based level-of-detail info
Y
Yu Yang 已提交
345
             LoD new_lod;
346 347
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
348 349
             PADDLE_ENFORCE(CheckLoD(new_lod, vectorize(self.dims()).front()),
                            "the provided lod info is invalid");
350
             self.set_lod(new_lod);
D
dangqingqing 已提交
351
           })
352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376
      .def("set_recursive_sequence_lengths",
           [](LoDTensor &self, const std::vector<std::vector<size_t>>
                                   &recursive_sequence_lengths) {
             // the input recursive_sequence_lengths is length-based
             // level-of-detail info
             LoD new_lod;
             new_lod.reserve(recursive_sequence_lengths.size());
             std::copy(recursive_sequence_lengths.begin(),
                       recursive_sequence_lengths.end(),
                       std::back_inserter(new_lod));
             LoD new_offset_lod = ConvertToOffsetBasedLoD(new_lod);
             PADDLE_ENFORCE(
                 CheckLoD(new_offset_lod, vectorize(self.dims()).front()),
                 "the provided recursive_sequence_lengths info is invalid");
             self.set_lod(new_offset_lod);
           })
      .def("lod",
           [](LoDTensor &self) -> std::vector<std::vector<size_t>> {
             // output the offset-based lod info
             LoD lod = self.lod();
             std::vector<std::vector<size_t>> new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             return new_lod;
           })
G
gongweibao 已提交
377
      // Set above comments of set_lod.
378 379 380 381 382 383 384 385 386 387 388 389 390
      .def("recursive_sequence_lengths",
           [](LoDTensor &self) -> std::vector<std::vector<size_t>> {
             // output the length-based lod info
             LoD lod = ConvertToLengthBasedLoD(self.lod());
             std::vector<std::vector<size_t>> new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             return new_lod;
           })
      .def("has_valid_recursive_sequence_lengths", [](LoDTensor &self) -> bool {
        // Check that the lod info is valid and match the outermost
        // dimension of the LoDTensor data
        return CheckLoD(self.lod(), vectorize(self.dims()).front());
D
dangqingqing 已提交
391 392
      });

Q
qijun 已提交
393 394 395 396 397 398 399 400 401 402 403
  py::class_<SelectedRows>(m, "SelectedRows")
      .def("__init__",
           [](SelectedRows &instance) { new (&instance) SelectedRows(); })
      .def("__init__",
           [](SelectedRows &instance, const std::vector<int64_t> rows,
              const int64_t &height) {
             new (&instance) SelectedRows(rows, height);
           })
      .def("get_tensor",
           [](SelectedRows &self) { return self.mutable_value(); },
           py::return_value_policy::reference)
404 405
      .def("numel",
           [](SelectedRows &self) -> int64_t { return self.value().numel(); })
Q
qijun 已提交
406 407
      .def("set_height", &SelectedRows::set_height)
      .def("height", &SelectedRows::height)
Q
qijun 已提交
408 409 410 411 412 413 414 415 416
      .def("set_rows",
           [](SelectedRows &self, std::vector<int64_t> rows) {
#ifndef PADDLE_WITH_CUDA
             self.set_rows(rows);
#else
        Vector<int64_t> new_rows(rows);
        self.set_rows(new_rows);
#endif
           })
417
      .def("sync_index", [](SelectedRows &instance) { instance.SyncIndex(); })
418
      .def("rows", [](SelectedRows &self) {
419 420 421 422 423
        auto rows = self.rows();
        std::vector<int64_t> new_rows;
        new_rows.reserve(rows.size());
        std::copy(rows.begin(), rows.end(), std::back_inserter(new_rows));
        return new_rows;
424
      });
Q
qijun 已提交
425

426
  py::class_<Variable>(m, "Variable", R"DOC(Variable Class.
427 428 429

All parameter, weight, gradient are variables in Paddle.
)DOC")
430
      .def("is_int", [](const Variable &var) { return var.IsType<int>(); })
431
      .def("set_int",
432 433
           [](Variable &var, int val) -> void { *var.GetMutable<int>() = val; })
      .def("get_int", [](const Variable &var) -> int { return var.Get<int>(); })
434 435 436 437 438 439 440
      .def("is_float", [](const Variable &var) { return var.IsType<float>(); })
      .def("set_float",
           [](Variable &var, float val) -> void {
             *var.GetMutable<float>() = val;
           })
      .def("get_float",
           [](const Variable &var) -> float { return var.Get<float>(); })
Y
Yu Yang 已提交
441
      .def("get_tensor",
442 443
           [](Variable &self) -> LoDTensor * {
             return self.GetMutable<LoDTensor>();
D
dangqingqing 已提交
444 445
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
446 447 448
      .def("get_lod_rank_table",
           [](Variable &self) { return self.GetMutable<LoDRankTable>(); },
           py::return_value_policy::reference)
Q
qijun 已提交
449 450 451 452 453
      .def("get_selected_rows",
           [](Variable &self) -> SelectedRows * {
             return self.GetMutable<SelectedRows>();
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
454 455 456
      .def("get_lod_tensor_array",
           [](Variable &self) { return self.GetMutable<LoDTensorArray>(); },
           py::return_value_policy::reference)
P
peizhilin 已提交
457
#if (defined(PADDLE_WITH_CUDA) && !defined(_WIN32))
D
Dong Zhihong 已提交
458 459 460 461 462
      .def("get_communicator",
           [](Variable &self) -> platform::Communicator * {
             return self.GetMutable<platform::Communicator>();
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
463
#endif
Y
Refine  
Yu Yang 已提交
464 465 466 467 468
      .def("get_reader",
           [](Variable &self) -> framework::ReaderHolder * {
             PADDLE_ENFORCE(self.IsType<framework::ReaderHolder>());
             return self.GetMutable<framework::ReaderHolder>();
           },
W
wopeizl 已提交
469
           py::return_value_policy::reference);
470

Y
Refine  
Yu Yang 已提交
471
  py::class_<framework::ReaderHolder>(m, "Reader", "")
472
      .def("reset", &framework::ReaderHolder::ResetAll);
Y
Refine  
Yu Yang 已提交
473

S
sneaxiy 已提交
474 475 476 477
  using LoDTensorBlockingQueue =
      ::paddle::operators::reader::LoDTensorBlockingQueue;
  using LoDTensorBlockingQueueHolder =
      ::paddle::operators::reader::LoDTensorBlockingQueueHolder;
S
sneaxiy 已提交
478 479
  py::class_<LoDTensorBlockingQueue, std::shared_ptr<LoDTensorBlockingQueue>>(
      m, "LoDTensorBlockingQueue", "")
S
sneaxiy 已提交
480
      .def("push",
S
sneaxiy 已提交
481
           [](LoDTensorBlockingQueue &self,
S
sneaxiy 已提交
482
              const std::vector<framework::LoDTensor> &lod_tensor_vec) {
S
sneaxiy 已提交
483
             pybind11::gil_scoped_release release;
S
sneaxiy 已提交
484
             return self.Push(lod_tensor_vec);
S
sneaxiy 已提交
485
           })
S
sneaxiy 已提交
486 487 488 489
      .def("size", &LoDTensorBlockingQueue::Size)
      .def("capacity", &LoDTensorBlockingQueue::Cap)
      .def("close", &LoDTensorBlockingQueue::Close)
      .def("is_closed", &LoDTensorBlockingQueue::IsClosed);
S
sneaxiy 已提交
490

S
sneaxiy 已提交
491
  m.def("init_lod_tensor_blocking_queue",
S
sneaxiy 已提交
492
        [](Variable &var, size_t capacity,
S
sneaxiy 已提交
493
           const std::vector<std::vector<int64_t>> &shapes)
S
sneaxiy 已提交
494
            -> std::shared_ptr<LoDTensorBlockingQueue> {
S
sneaxiy 已提交
495 496 497 498 499 500
              std::vector<DDim> dims(shapes.size());
              std::transform(shapes.begin(), shapes.end(), dims.begin(),
                             [](const std::vector<int64_t> &shape) {
                               return make_ddim(shape);
                             });
              auto *holder = var.GetMutable<LoDTensorBlockingQueueHolder>();
501 502
              holder->InitOnce(capacity, dims,
                               FLAGS_reader_queue_speed_test_mode);
S
sneaxiy 已提交
503
              return holder->GetQueue();
S
sneaxiy 已提交
504
            },
S
sneaxiy 已提交
505
        py::return_value_policy::copy);
S
sneaxiy 已提交
506

S
sneaxiy 已提交
507
  py::class_<Scope>(m, "_Scope", R"DOC(
Q
Qiao Longfei 已提交
508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526
    Scope is an association of a name to Variable. All variables belong to Scope.

    Variables in a parent scope can be retrieved from local scope.

    You need to specify a scope to run a Net, i.e., `exe.Run(&scope)`.
    One net can run in different scopes and update different variable in the
    scope.

    You can create var in a scope and get it from the scope.

    Examples:
        .. code-block:: python

          # create tensor from a scope and set value to it.
          param = scope.var('Param').get_tensor()
          param_array = np.full((height, row_numel), 5.0).astype("float32")
          param.set(param_array, place)

        )DOC")
S
sneaxiy 已提交
527 528
      .def("_remove_from_pool",
           [](Scope &self) { ScopePool::Instance().Remove(&self); })
D
dongzhihong 已提交
529
      .def("var",
530
           [](Scope &self, const std::string &name) -> Variable * {
D
dongzhihong 已提交
531
             return self.Var(name);
Y
Yu Yang 已提交
532
           },
533
           py::return_value_policy::reference)
534
      .def("find_var", &Scope::FindVar, py::return_value_policy::reference)
535
      .def("new_scope", [](Scope &self) -> Scope * { return &self.NewScope(); },
536
           py::return_value_policy::reference)
Y
Yu Yang 已提交
537
      .def("drop_kids", &Scope::DropKids);
538

S
sneaxiy 已提交
539 540 541 542 543 544 545 546
  m.def("Scope",
        []() -> Scope * {
          auto *s = new Scope();
          ScopePool::Instance().Insert(std::unique_ptr<Scope>(s));
          return s;
        },
        py::return_value_policy::reference);

Y
Yu Yang 已提交
547 548
  //! @note: Be careful! PyBind will return std::string as an unicode, not
  //! Python str. If you want a str object, you should cast them in Python.
Y
Yu Yang 已提交
549 550
  m.def("get_all_op_protos", []() -> std::vector<py::bytes> {
    std::vector<py::bytes> ret_values;
551 552 553 554 555 556 557 558 559 560
    for (auto &iter : OpInfoMap::Instance().map()) {
      auto &info = iter.second;
      if (info.HasOpProtoAndChecker()) {
        std::string str;
        PADDLE_ENFORCE(
            info.Proto().SerializeToString(&str),
            "Serialize OpProto Error. This could be a bug of Paddle.");
        ret_values.emplace_back(str);
      }
    }
Y
Yu Yang 已提交
561 562
    return ret_values;
  });
563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578
  m.def(
      "get_grad_op_desc", [](const OpDesc &op_desc,
                             const std::unordered_set<std::string> &no_grad_set,
                             const std::vector<BlockDesc *> &grad_sub_block) {
        std::unordered_map<std::string, std::string> grad_to_var;
        std::vector<std::unique_ptr<OpDesc>> grad_op_descs =
            framework::OpInfoMap::Instance()
                .Get(op_desc.Type())
                .GradOpMaker()(op_desc, no_grad_set, &grad_to_var,
                               grad_sub_block);
        std::vector<OpDesc *> grad_op_desc_ptrs(grad_op_descs.size());
        std::transform(grad_op_descs.begin(), grad_op_descs.end(),
                       grad_op_desc_ptrs.begin(),
                       [](std::unique_ptr<OpDesc> &p) { return p.release(); });
        return std::make_pair(grad_op_desc_ptrs, grad_to_var);
      });
Y
Yu Yang 已提交
579
  m.def("prune", [](const ProgramDesc &origin,
580
                    const std::vector<std::array<size_t, 2>> &targets) {
Y
Yu Yang 已提交
581
    ProgramDesc prog_with_targets(origin);
582
    for (const auto &t : targets) {
583
      prog_with_targets.MutableBlock(t[0])->Op(t[1])->SetIsTarget(true);
584
    }
585
    proto::ProgramDesc pruned_desc;
586
    Prune(*prog_with_targets.Proto(), &pruned_desc);
Y
Yu Yang 已提交
587
    return new ProgramDesc(pruned_desc);
588
  });
589 590 591 592
  m.def("empty_var_name",
        []() { return std::string(framework::kEmptyVarName); });
  m.def("grad_var_suffix",
        []() { return std::string(framework::kGradVarSuffix); });
593 594 595
  m.def_submodule(
       "var_names",
       "The module will return special predefined variable name in Paddle")
Y
Yi Wang 已提交
596 597
      .def("empty", []() { return kEmptyVarName; })
      .def("temp", []() { return kTempVarName; });
Q
qijun 已提交
598
  // clang-format off
Y
Yu Yang 已提交
599
  py::class_<paddle::platform::DeviceContext>(m, "DeviceContext")
Q
qijun 已提交
600 601
      .def_static("create",
                  [](paddle::platform::CPUPlace& place)
Q
qijun 已提交
602
                      -> paddle::platform::DeviceContext* {
Q
qijun 已提交
603 604 605
                    return new paddle::platform::CPUDeviceContext();
                  })
      .def_static("create",
D
dzhwinter 已提交
606
                  [](paddle::platform::CUDAPlace& place)
Q
qijun 已提交
607
                      -> paddle::platform::DeviceContext* {
608
#ifndef PADDLE_WITH_CUDA
D
dzhwinter 已提交
609
                    PADDLE_THROW("CUDAPlace is not supported in CPU device.");
Q
qijun 已提交
610
#else
Q
qijun 已提交
611
                    return new paddle::platform::CUDADeviceContext(place);
Q
qijun 已提交
612
#endif
C
chengduoZH 已提交
613 614 615 616 617 618 619 620 621 622 623
                  })
          .def_static("create",
                [](paddle::platform::CUDAPinnedPlace& place)
                        -> paddle::platform::DeviceContext* {
#ifndef PADDLE_WITH_CUDA
                  PADDLE_THROW(
                        "CUDAPinnedPlace is not supported in CPU device.");
#else
                  return new paddle::platform::CUDAPinnedDeviceContext(place);
#endif
                });;
D
Dong Zhihong 已提交
624
// clang-format on
P
peizhilin 已提交
625
#if (defined(PADDLE_WITH_CUDA) && !defined(_WIN32))
D
Dong Zhihong 已提交
626 627
  py::class_<platform::Communicator>(m, "Communicator").def(py::init<>());
#endif
D
dzhwinter 已提交
628
  py::class_<platform::CUDAPlace>(m, "CUDAPlace")
629
      .def(py::init<int>())
D
dzhwinter 已提交
630
      .def("__str__", string::to_string<const platform::CUDAPlace &>);
Q
qijun 已提交
631

632 633 634
  py::class_<paddle::platform::CPUPlace>(m, "CPUPlace")
      .def(py::init<>())
      .def("__str__", string::to_string<const platform::CPUPlace &>);
Y
Yu Yang 已提交
635

C
chengduoZH 已提交
636 637 638 639
  py::class_<paddle::platform::CUDAPinnedPlace>(m, "CUDAPinnedPlace")
      .def(py::init<>())
      .def("__str__", string::to_string<const platform::CUDAPinnedPlace &>);

Y
Yu Yang 已提交
640 641 642 643 644 645 646
  py::class_<platform::Place>(m, "Place")
      .def(py::init<>())
      .def("set_place",
           [](platform::Place &self, const platform::CPUPlace &cpu_place) {
             self = cpu_place;
           })
      .def("set_place",
D
dzhwinter 已提交
647
           [](platform::Place &self, const platform::CUDAPlace &gpu_place) {
Y
Yu Yang 已提交
648
             self = gpu_place;
C
chengduoZH 已提交
649 650
           })
      .def("set_place", [](platform::Place &self,
C
chengduoZH 已提交
651 652
                           const platform::CUDAPinnedPlace &cuda_pinned_place) {
        self = cuda_pinned_place;
C
chengduoZH 已提交
653
      });
Y
Yu Yang 已提交
654

Y
Yu Yang 已提交
655 656 657
  py::class_<OperatorBase>(m, "Operator")
      .def_static("create",
                  [](py::bytes protobin) {
658
                    proto::OpDesc desc;
Y
Yu Yang 已提交
659 660 661 662 663
                    PADDLE_ENFORCE(desc.ParsePartialFromString(protobin),
                                   "Cannot parse user input to OpDesc");
                    PADDLE_ENFORCE(desc.IsInitialized(),
                                   "User OpDesc is not initialized, reason %s",
                                   desc.InitializationErrorString());
664
                    return OpRegistry::CreateOp(desc);
Y
Yu Yang 已提交
665
                  })
666
      .def("run",
667
           [](OperatorBase &self, const Scope &scope,
D
dzhwinter 已提交
668 669 670
              const platform::CPUPlace &place) { self.Run(scope, place); })
      .def("run",
           [](OperatorBase &self, const Scope &scope,
D
dzhwinter 已提交
671
              const platform::CUDAPlace &place) { self.Run(scope, place); })
C
chengduoZH 已提交
672 673 674 675 676
      .def("run",
           [](OperatorBase &self, const Scope &scope,
              const platform::CUDAPinnedPlace &place) {
             self.Run(scope, place);
           })
Y
Yu Yang 已提交
677 678 679 680 681 682 683
      .def("type",
           [](const OperatorBase &op) -> std::string { return op.Type(); })
      .def("outputs",
           [](const OperatorBase &op)
               -> std::map<std::string, std::vector<std::string>> {
                 return op.Outputs();
               })
Q
qijun 已提交
684 685
      .def("output_vars",
           [](const OperatorBase &op) { return op.OutputVars(true); })
Y
Yu Yang 已提交
686
      .def("inputs", [](const OperatorBase &op) { return op.Inputs(); })
Q
qijun 已提交
687
      .def("input_vars", [](const OperatorBase &op) { return op.InputVars(); })
Y
Yu Yang 已提交
688 689 690 691
      .def("__str__", &OperatorBase::DebugString)
      .def("no_intermediate_outputs",
           [](const OperatorBase &op) { return op.OutputVars(false); })
      .def("support_gpu", &OperatorBase::SupportGPU);
Y
Yu Yang 已提交
692

F
fengjiayi 已提交
693
  py::class_<framework::Executor>(m, "Executor")
D
dzhwinter 已提交
694
      .def(py::init<const platform::Place &>())
Y
Yancey1989 已提交
695
      .def("close", &Executor::Close)
S
sneaxiy 已提交
696 697 698 699 700
      .def("run", [](Executor &self, const ProgramDesc &prog, Scope *scope,
                     int block_id, bool create_local_scope, bool create_vars) {
        pybind11::gil_scoped_release release;
        self.Run(prog, scope, block_id, create_local_scope, create_vars);
      });
S
sneaxiy 已提交
701

D
dzhwinter 已提交
702
  m.def("init_gflags", framework::InitGflags);
Y
Yang Yu 已提交
703
  m.def("init_glog", framework::InitGLOG);
X
Xin Pan 已提交
704 705
  m.def("init_devices",
        [](bool init_p2p) { framework::InitDevices(init_p2p); });
706

707
  m.def("is_compiled_with_cuda", IsCompiledWithCUDA);
708
  m.def("is_compiled_with_brpc", IsCompiledWithBrpc);
Y
update  
Yancey1989 已提交
709
  m.def("is_compiled_with_dist", IsCompiledWithDIST);
710 711 712 713 714 715
#ifdef PADDLE_WITH_CUDA
  m.def("is_float16_supported", [](const platform::CUDAPlace &place) -> bool {
    // Only GPUs with Compute Capability >= 53 support float16
    return platform::GetCUDAComputeCapability(place.device) >= 53;
  });
#endif
716

717
  m.def("set_feed_variable", framework::SetFeedVariable);
Q
qijun 已提交
718
  m.def("get_fetch_variable", framework::GetFetchVariable);
719
  m.def("get_variable_tensor", framework::GetVariableTensor);
Q
qijun 已提交
720

X
Xin Pan 已提交
721 722
  m.def("_is_program_version_supported", IsProgramVersionSupported);

723 724 725 726 727
  BindProgramDesc(&m);
  BindBlockDesc(&m);
  BindVarDsec(&m);
  BindOpDesc(&m);
  BindConstValue(&m);
Y
Yu Yang 已提交
728

Y
Yu Yang 已提交
729 730 731 732 733 734 735 736 737
  py::class_<framework::LoDRankTable>(m, "LodRankTable")
      .def("items", [](framework::LoDRankTable &table) {
        std::vector<std::pair<size_t, size_t>> res;
        for (auto &item : table.items()) {
          res.push_back({item.index, item.length});
        }
        return res;
      });

Y
Yu Yang 已提交
738
  py::class_<LoDTensorArray>(m, "LoDTensorArray")
S
sneaxiy 已提交
739 740
      .def("__init__",
           [](LoDTensorArray &instance) { new (&instance) LoDTensorArray(); })
Y
Yu Yang 已提交
741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756
      .def("__getitem__",
           [](LoDTensorArray &self, size_t i) { return &self.at(i); },
           py::return_value_policy::reference)
      .def("__len__", [](LoDTensorArray &self) { return self.size(); })
      .def("__setitem__",
           [](LoDTensorArray &self, size_t i, const LoDTensor &t) {
             PADDLE_ENFORCE_LT(i, self.size());
             self[i].ShareDataWith(t);
             self[i].set_lod(t.lod());
           })
      .def("append", [](LoDTensorArray &self, const LoDTensor &t) {
        self.emplace_back();
        self.back().ShareDataWith(t);
        self.back().set_lod(t.lod());
      });

D
dzhwinter 已提交
757 758 759
  m.def("IsInplace",
        [](std::string op) -> bool { return operators::IsInplace(op); });

Y
Yu Yang 已提交
760
  m.def("op_support_gpu", OpSupportGPU);
D
Dong Zhihong 已提交
761
#ifdef PADDLE_WITH_CUDA
D
Dong Zhihong 已提交
762
  m.def("get_cuda_device_count", platform::GetCUDADeviceCount);
D
dangqingqing 已提交
763

P
peizhilin 已提交
764
#ifndef _WIN32
D
dangqingqing 已提交
765 766 767
  m.def("nvprof_init", platform::CudaProfilerInit);
  m.def("nvprof_start", platform::CudaProfilerStart);
  m.def("nvprof_stop", platform::CudaProfilerStop);
D
Dong Zhihong 已提交
768
#endif
P
peizhilin 已提交
769
#endif
Y
Yu Yang 已提交
770

771 772 773 774
  py::enum_<platform::ProfilerState>(m, "ProfilerState", py::arithmetic())
      .value("kDisabled", platform::ProfilerState::kDisabled)
      .value("kCPU", platform::ProfilerState::kCPU)
      .value("kCUDA", platform::ProfilerState::kCUDA)
775
      .value("kAll", platform::ProfilerState::kAll)
776 777 778 779 780 781 782 783 784 785 786 787 788
      .export_values();

  py::enum_<platform::EventSortingKey>(m, "EventSortingKey", py::arithmetic())
      .value("kDefault", platform::EventSortingKey::kDefault)
      .value("kCalls", platform::EventSortingKey::kCalls)
      .value("kTotal", platform::EventSortingKey::kTotal)
      .value("kMin", platform::EventSortingKey::kMin)
      .value("kMax", platform::EventSortingKey::kMax)
      .value("kAve", platform::EventSortingKey::kAve)
      .export_values();

  m.def("enable_profiler", platform::EnableProfiler);
  m.def("disable_profiler", platform::DisableProfiler);
X
Xin Pan 已提交
789
  m.def("is_profiler_enabled", platform::IsProfileEnabled);
790
  m.def("reset_profiler", platform::ResetProfiler);
Y
Yu Yang 已提交
791

792 793
  py::class_<ir::Pass, std::shared_ptr<ir::Pass>> pass(m, "Pass");
  pass.def(py::init())
794 795 796 797 798
      .def(
          "set_str",
          [](ir::Pass &self, const std::string &name, const std::string &attr) {
            self.Set<std::string>(name, new std::string(attr));
          })
X
Xin Pan 已提交
799 800
      .def("set_int", [](ir::Pass &self, const std::string &name,
                         int val) { self.Set<const int>(name, new int(val)); })
F
flame 已提交
801 802 803 804 805 806
      .def("type", &ir::Pass::Type)
      .def("apply", [](ir::Pass &self, std::shared_ptr<ir::Graph> graph) {
        std::unique_ptr<ir::Graph> origin_graph(graph.get());
        auto optim_graph = self.Apply(std::move(origin_graph));
        graph.reset(optim_graph.release());
      });
807

X
fix  
Xin Pan 已提交
808 809
  py::class_<ir::PassBuilder, std::shared_ptr<ir::PassBuilder>> pb(
      m, "PassBuilder");
810 811 812 813 814 815 816 817 818 819 820 821 822 823
  pb.def(py::init())
      .def("append_pass",
           [](ir::PassBuilder &self,
              const std::string &pass_type) -> std::shared_ptr<ir::Pass> {
             return self.AppendPass(pass_type);
           })
      .def("all_passes", [](ir::PassBuilder &self) { return self.AllPasses(); })
      .def("insert_pass",
           [](ir::PassBuilder &self, size_t idx, const std::string &pass_type) {
             return self.InsertPass(idx, pass_type);
           })
      .def("remove_pass",
           [](ir::PassBuilder &self, size_t idx) { self.RemovePass(idx); });

Y
yuyang18 已提交
824
  // -- python binds for parallel executor.
Y
yuyang18 已提交
825
  py::class_<ParallelExecutor> pe(m, "ParallelExecutor");
C
chengduo 已提交
826 827 828 829
  py::class_<ExecutionStrategy> exec_strategy(pe, "ExecutionStrategy", R"DOC(
    ExecutionStrategy allows the user to more preciously control how to run
    the program in ParallelExecutor by setting the property.

C
chengduo 已提交
830 831 832 833 834 835 836 837 838 839 840
    Examples:
        .. code-block:: python

          exec_strategy = fluid.ExecutionStrategy()
          exec_strategy.num_threads = 4

          train_exe = fluid.ParallelExecutor(use_cuda=True,
                                             loss_name=loss.name,
                                             exec_strategy=exec_strategy)

          train_loss, = train_exe.run([loss.name], feed=feed_dict)
C
chengduo 已提交
841 842 843

        )DOC");

Y
yuyang18 已提交
844
  exec_strategy.def(py::init())
Y
yuyang18 已提交
845 846 847 848 849
      .def_property(
          "num_threads",
          [](const ExecutionStrategy &self) { return self.num_threads_; },
          [](ExecutionStrategy &self, size_t num_threads) {
            self.num_threads_ = num_threads;
C
chengduo 已提交
850 851 852 853 854 855 856 857 858 859
          },
          R"DOC(The type is INT, num_threads represents the size of thread pool that
            used to run the operators of the current program in ParallelExecutor.
            If :math:`num\_threads=1`, all the operators will execute one by one,
            but the order maybe difference between iterations.
            If it is not set, it will be set in ParallelExecutor according to the
            device type and device count, for GPU, :math:`num\_threads=device\_count*4`, for CPU,
            :math:`num\_threads=CPU\_NUM*4`, the explanation of:math:`CPU\_NUM` is in ParallelExecutor.
            if it is not set, ParallelExecutor will get the cpu count by calling
            `multiprocessing.cpu_count()`. Default 0.)DOC")
Y
yuyang18 已提交
860
      .def_property(
861 862 863 864
          "use_cuda",
          [](const ExecutionStrategy &self) { return self.use_cuda_; },
          [](ExecutionStrategy &self, bool use_cuda) {
            self.use_cuda_ = use_cuda;
C
chengduo 已提交
865 866 867 868
          })  // FIXME(chengduo): Doesn't add doc for 'use_cuda', use_cuda may
      // make user confuse, because ParallelExecutor has a parameter named
      // 'use_cuda' too, in current implementation, ParallelExecutor's
      // 'use_cuda' will rewrite ExecutionStrategy's 'use_cuda'.
Y
yuyang18 已提交
869 870 871 872 873
      .def_property(
          "allow_op_delay",
          [](const ExecutionStrategy &self) { return self.allow_op_delay_; },
          [](ExecutionStrategy &self, bool allow_op_delay) {
            self.allow_op_delay_ = allow_op_delay;
C
chengduo 已提交
874 875 876 877
          },
          R"DOC(The type is BOOL, allow_op_delay represents whether to delay the
                communication operators to run, it may make the execution faster.
                Note that in some models, allow_op_delay may cause program hang. Default False.)DOC")
Y
yuyang18 已提交
878 879 880 881 882 883 884
      .def_property(
          "num_iteration_per_drop_scope",
          [](const ExecutionStrategy &self) {
            return self.num_iteration_per_drop_scope_;
          },
          [](ExecutionStrategy &self, size_t num_iteration_per_drop_scope) {
            self.num_iteration_per_drop_scope_ = num_iteration_per_drop_scope;
C
chengduo 已提交
885 886 887 888 889 890 891 892 893 894 895
          },
          R"DOC(The type is INT, num_iteration_per_drop_scope indicates how
                many iterations to clean up the temp variables which
                is generated during execution. It may make the execution faster,
                because the temp variable's shape maybe the same between two iterations. Default 100.

                NOTES:
                    1. If you fetch data when calling the 'run', the ParallelExecutor
                       will clean up the temp variables at the end of the current iteration.
                    2. In some NLP model, it may cause the GPU memory is insufficient,
                       in this case, you should reduce `num_iteration_per_drop_scope`.
896 897 898 899 900 901
              )DOC")
      .def_property("_dry_run",
                    [](const ExecutionStrategy &self) { return self.dry_run_; },
                    [](ExecutionStrategy &self, bool dry_run) {
                      self.dry_run_ = dry_run;
                    });
C
chengduo 已提交
902

Y
yuyang18 已提交
903
  exec_strategy.def_property(
Y
yuyang18 已提交
904 905 906 907 908 909 910
      "use_experimental_executor",
      [](const ExecutionStrategy &self) {
        return self.type_ == ExecutionStrategy::kExperimental;
      },
      [](ExecutionStrategy &self, bool experimental) {
        self.type_ = experimental ? ExecutionStrategy::kExperimental
                                  : ExecutionStrategy::kDefault;
Y
yuyang18 已提交
911 912
      });

C
chengduo 已提交
913 914 915 916
  py::class_<BuildStrategy> build_strategy(pe, "BuildStrategy", R"DOC(
    BuildStrategy allows the user to more preciously control how to
    build the SSA Graph in ParallelExecutor by setting the property.

C
chengduo 已提交
917 918 919 920 921 922 923 924 925 926 927
    Examples:
        .. code-block:: python

          build_strategy = fluid.BuildStrategy()
          build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce

          train_exe = fluid.ParallelExecutor(use_cuda=True,
                                             loss_name=loss.name,
                                             build_strategy=build_strategy)

          train_loss, = train_exe.run([loss.name], feed=feed_dict)
C
chengduo 已提交
928
)DOC");
Y
yuyang18 已提交
929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944

  py::enum_<BuildStrategy::ReduceStrategy>(build_strategy, "ReduceStrategy")
      .value("Reduce", BuildStrategy::ReduceStrategy::kReduce)
      .value("AllReduce", BuildStrategy::ReduceStrategy::kAllReduce);
  py::enum_<BuildStrategy::GradientScaleStrategy>(build_strategy,
                                                  "GradientScaleStrategy")
      .value("CoeffNumDevice",
             BuildStrategy::GradientScaleStrategy::kCoeffNumDevice)
      .value("One", BuildStrategy::GradientScaleStrategy::kOne)
      .value("Customized", BuildStrategy::GradientScaleStrategy::kCustomized);

  build_strategy.def(py::init())
      .def_property(
          "reduce_strategy",
          [](const BuildStrategy &self) { return self.reduce_; },
          [](BuildStrategy &self, BuildStrategy::ReduceStrategy strategy) {
X
Xin Pan 已提交
945
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
Y
yuyang18 已提交
946
            self.reduce_ = strategy;
C
chengduo 已提交
947 948 949 950 951 952 953
          },
          R"DOC(The type is STR, there are two reduce strategies in ParallelExecutor,
                  'AllReduce' and 'Reduce'. If you want that all the parameters'
                  optimization are done on all devices independently, you should choose 'AllReduce';
                  if you choose 'Reduce', all the parameters' optimization will be evenly distributed
                  to different devices, and then broadcast the optimized parameter to other devices.
                  In some models, `Reduce` is faster. Default 'AllReduce'. )DOC")
Y
yuyang18 已提交
954 955 956 957 958
      .def_property(
          "gradient_scale_strategy",
          [](const BuildStrategy &self) { return self.gradient_scale_; },
          [](BuildStrategy &self,
             BuildStrategy::GradientScaleStrategy strategy) {
X
Xin Pan 已提交
959
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
Y
yuyang18 已提交
960
            self.gradient_scale_ = strategy;
C
chengduo 已提交
961 962 963 964 965 966
          },
          R"DOC(The type is STR, there are three ways of defining :math:`loss@grad` in
                   ParallelExecutor, 'CoeffNumDevice', 'One' and 'Customized'. By default,
                   ParallelExecutor sets the :math:`loss@grad` according to the number of devices.
                   If you want to customize :math:`loss@grad`, you can choose 'Customized'.
                   Default 'CoeffNumDevice'.)DOC")
Y
yuyang18 已提交
967 968 969 970
      .def_property(
          "debug_graphviz_path",
          [](const BuildStrategy &self) { return self.debug_graphviz_path_; },
          [](BuildStrategy &self, const std::string &path) {
X
Xin Pan 已提交
971
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
Y
yuyang18 已提交
972
            self.debug_graphviz_path_ = path;
C
chengduo 已提交
973 974 975 976
          },
          R"DOC(The type is STR, debug_graphviz_path indicate the path that
                    writing the SSA Graph to file in the form of graphviz, you.
                    It is useful for debugging. Default "")DOC")
S
sneaxiy 已提交
977 978 979 980 981 982
      .def_property(
          "enable_sequential_execution",
          [](const BuildStrategy &self) {
            return self.enable_sequential_execution_;
          },
          [](BuildStrategy &self, bool b) {
X
Xin Pan 已提交
983
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
S
sneaxiy 已提交
984 985 986 987 988 989 990 991 992
            self.enable_sequential_execution_ = b;
          },
          R"DOC(The type is BOOL. If set True, the execution order of ops would be the same as what is in the program. Default False.)DOC")
      .def_property(
          "remove_unnecessary_lock",
          [](const BuildStrategy &self) {
            return self.remove_unnecessary_lock_;
          },
          [](BuildStrategy &self, bool b) {
X
Xin Pan 已提交
993
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
S
sneaxiy 已提交
994 995 996
            self.remove_unnecessary_lock_ = b;
          },
          R"DOC(The type is BOOL. If set True, some locks in GPU ops would be released and ParallelExecutor would run faster. Default False.)DOC")
997 998 999 1000 1001 1002
      .def_property(
          "num_trainers",
          [](const BuildStrategy &self) { return self.num_trainers_; },
          [](BuildStrategy &self, int num_trainers) {
            self.num_trainers_ = num_trainers;
          })
1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014
      .def_property(
          "trainers_endpoints",
          [](const BuildStrategy &self) { return self.trainers_endpoints_; },
          [](BuildStrategy &self,
             const std::vector<std::string> &trainers_endpoints) {
            self.trainers_endpoints_ = trainers_endpoints;
          })
      .def_property("trainer_id",
                    [](const BuildStrategy &self) { return self.trainer_id_; },
                    [](BuildStrategy &self, int trainer_id) {
                      self.trainer_id_ = trainer_id;
                    })
C
chengduo 已提交
1015 1016 1017 1018 1019 1020
      .def_property(
          "fuse_elewise_add_act_ops",
          [](const BuildStrategy &self) {
            return self.fuse_elewise_add_act_ops_;
          },
          [](BuildStrategy &self, bool b) {
X
Xin Pan 已提交
1021
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
C
chengduo 已提交
1022 1023 1024 1025 1026
            self.fuse_elewise_add_act_ops_ = b;
          },
          R"DOC(The type is BOOL, fuse_elewise_add_act_ops indicate whether
                     to fuse elementwise_add_op and activation_op,
                     it may make the execution faster. Default False)DOC")
1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040
      .def_property(
          "fuse_relu_depthwise_conv",
          [](const BuildStrategy &self) {
            return self.fuse_relu_depthwise_conv_;
          },
          [](BuildStrategy &self, bool b) {
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
            self.fuse_relu_depthwise_conv_ = b;
          },
          R"DOC(The type is BOOL, fuse_relu_depthwise_conv indicate whether
                      to fuse relu and depthwise_conv2d,
                      it will save GPU memory and may make the execution faster.
                      This options is only available in GPU devices.
                      Default False)DOC")
D
dzhwinter 已提交
1041 1042 1043 1044
      .def_property(
          "memory_optimize",
          [](const BuildStrategy &self) { return self.memory_optimize_; },
          [](BuildStrategy &self, bool b) { self.memory_optimize_ = b; })
1045 1046 1047 1048
      .def_property(
          "is_distribution",
          [](const BuildStrategy &self) { return self.is_distribution_; },
          [](BuildStrategy &self, bool b) { self.is_distribution_ = b; })
D
dzhwinter 已提交
1049 1050 1051 1052
      .def_property(
          "memory_early_delete",
          [](const BuildStrategy &self) { return self.memory_early_delete_; },
          [](BuildStrategy &self, bool b) { self.memory_early_delete_ = b; })
1053
      .def("_finalize_strategy_and_create_passes",
X
fix  
Xin Pan 已提交
1054
           [](BuildStrategy &self) -> std::shared_ptr<ir::PassBuilder> {
1055 1056 1057 1058 1059
             return self.CreatePassesFromStrategy(true);
           },
           R"DOC(Allow user to customized passes. Normally model-specific
                optimization passes should be defined in this way. BuildStrategy
                cannot be updated after being finalized.)DOC");
Y
yuyang18 已提交
1060 1061 1062

  pe.def(py::init<const std::vector<platform::Place> &,
                  const std::unordered_set<std::string> &, const ProgramDesc &,
Y
yuyang18 已提交
1063
                  const std::string &, Scope *, std::vector<Scope *> &,
1064
                  const ExecutionStrategy &, const BuildStrategy &>())
Y
Yu Yang 已提交
1065 1066 1067 1068
      // NOTE: even we return a vec<Scope*>* to Python use reference policy.
      // We still cannot get local_scope from this vector, since the element
      // of vec<Scope*> will be freed by Python GC. We can only return Scope*
      // one by one and mark them as reference.
1069 1070 1071 1072 1073
      .def("local_scopes",
           [](ParallelExecutor &self) -> std::vector<Scope *> * {
             return &self.GetLocalScopes();
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
1074 1075 1076 1077
      .def("feed_tensors_into_local_scopes",
           &ParallelExecutor::FeedTensorsIntoLocalScopes)
      .def("feed_and_split_tensor_into_local_scopes",
           &ParallelExecutor::FeedAndSplitTensorIntoLocalScopes)
S
sneaxiy 已提交
1078 1079 1080 1081 1082 1083
      .def("run", [](ParallelExecutor &self,
                     const std::vector<std::string> &fetch_tensors,
                     const std::string &fetched_var_name) {
        pybind11::gil_scoped_release release;
        self.Run(fetch_tensors, fetched_var_name);
      });
Y
Yu Yang 已提交
1084

1085
  BindRecordIOWriter(&m);
W
Wang Guibao 已提交
1086
  BindAsyncExecutor(&m);
F
flame 已提交
1087 1088
  BindGraph(&m);
  BindNode(&m);
F
flame 已提交
1089
  BindInferenceApi(&m);
L
Luo Tao 已提交
1090
}
1091
}  // namespace pybind
1092
}  // namespace paddle