pybind.cc 45.6 KB
Newer Older
1
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
2 3 4 5 6

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

7
http://www.apache.org/licenses/LICENSE-2.0
8 9 10 11 12 13

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
L
lgone2000 已提交
14
#include <Python.h>
C
chengduoZH 已提交
15 16
#include <algorithm>
#include <map>
S
sneaxiy 已提交
17
#include <memory>
C
chengduoZH 已提交
18 19 20 21 22
#include <mutex>  // NOLINT // for call_once
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>
23

Y
Yi Wang 已提交
24 25 26
#include "paddle/fluid/framework/executor.h"
#include "paddle/fluid/framework/feed_fetch_method.h"
#include "paddle/fluid/framework/framework.pb.h"
27
#include "paddle/fluid/framework/ir/pass_builder.h"
Y
Yi Wang 已提交
28 29 30
#include "paddle/fluid/framework/lod_rank_table.h"
#include "paddle/fluid/framework/lod_tensor.h"
#include "paddle/fluid/framework/lod_tensor_array.h"
31
#include "paddle/fluid/framework/op_registry.h"
Y
Yu Yang 已提交
32
#include "paddle/fluid/framework/parallel_executor.h"
Y
Yi Wang 已提交
33
#include "paddle/fluid/framework/prune.h"
Y
Refine  
Yu Yang 已提交
34
#include "paddle/fluid/framework/reader.h"
S
sneaxiy 已提交
35
#include "paddle/fluid/framework/scope_pool.h"
Y
Yi Wang 已提交
36
#include "paddle/fluid/framework/selected_rows.h"
X
Xin Pan 已提交
37
#include "paddle/fluid/framework/version.h"
38
#include "paddle/fluid/imperative/layer.h"
Y
Refine  
Yu Yang 已提交
39
#include "paddle/fluid/memory/allocation/allocator_strategy.h"
D
dzhwinter 已提交
40
#include "paddle/fluid/operators/activation_op.h"
S
sneaxiy 已提交
41
#include "paddle/fluid/operators/py_func_op.h"
S
sneaxiy 已提交
42
#include "paddle/fluid/operators/reader/lod_tensor_blocking_queue.h"
Y
Yu Yang 已提交
43
#include "paddle/fluid/platform/cpu_info.h"
Y
Yi Wang 已提交
44
#include "paddle/fluid/platform/enforce.h"
45
#include "paddle/fluid/platform/init.h"
Y
Yi Wang 已提交
46 47
#include "paddle/fluid/platform/place.h"
#include "paddle/fluid/platform/profiler.h"
W
Wang Guibao 已提交
48
#include "paddle/fluid/pybind/async_executor_py.h"
Y
Yi Wang 已提交
49 50
#include "paddle/fluid/pybind/const_value.h"
#include "paddle/fluid/pybind/exception.h"
51
#include "paddle/fluid/pybind/imperative.h"
F
flame 已提交
52
#include "paddle/fluid/pybind/ir.h"
53 54
#include "paddle/fluid/pybind/protobuf.h"
#include "paddle/fluid/pybind/pybind.h"  // NOLINT
Y
Yu Yang 已提交
55
#include "paddle/fluid/pybind/recordio.h"
Y
Yi Wang 已提交
56
#include "paddle/fluid/pybind/tensor_py.h"
Y
Yu Yang 已提交
57

58
#include "paddle/fluid/string/to_string.h"
59

D
Dong Zhihong 已提交
60
#ifdef PADDLE_WITH_CUDA
P
peizhilin 已提交
61
#ifndef _WIN32
Y
Yi Wang 已提交
62
#include "paddle/fluid/operators/nccl/nccl_gpu_common.h"
P
peizhilin 已提交
63
#endif
Y
Yi Wang 已提交
64 65
#include "paddle/fluid/platform/cuda_profiler.h"
#include "paddle/fluid/platform/gpu_info.h"
D
Dong Zhihong 已提交
66 67
#endif

M
minqiyang 已提交
68 69
#include "pybind11/stl.h"

70 71 72 73
DEFINE_bool(reader_queue_speed_test_mode, false,
            "If set true, the queue.pop will only get data from queue but not "
            "remove the data from queue for speed testing");

Q
Qiao Longfei 已提交
74 75 76
// disable auto conversion to list in Python
PYBIND11_MAKE_OPAQUE(paddle::framework::LoDTensorArray);

77
namespace paddle {
78
namespace pybind {
79
bool IsCompiledWithCUDA() {
80
#ifndef PADDLE_WITH_CUDA
Q
qijun 已提交
81 82 83 84 85 86
  return false;
#else
  return true;
#endif
}

87
bool IsCompiledWithBrpc() {
88
#ifndef PADDLE_WITH_DISTRIBUTE
89 90
  return false;
#endif
91 92 93 94 95 96

#ifdef PADDLE_WITH_GRPC
  return false;
#endif

  return true;
97 98
}

Y
update  
Yancey1989 已提交
99
bool IsCompiledWithDIST() {
Y
Yancey1989 已提交
100
#ifdef PADDLE_WITH_DISTRIBUTE
Y
update  
Yancey1989 已提交
101 102 103 104 105 106
  return true;
#else
  return false;
#endif
}

107
PYBIND11_MODULE(core, m) {
Y
Yu Yang 已提交
108 109 110
  // Not used, just make sure cpu_info.cc is linked.
  paddle::platform::CpuTotalPhysicalMemory();

Y
Refine  
Yu Yang 已提交
111
  paddle::memory::allocation::UseAllocatorStrategyGFlag();
112
  m.doc() = "C++ core of PaddlePaddle";
113

114 115 116 117
  // using framework in this function. Since it is inside a function, it will
  // not cause namespace pollution.
  using namespace paddle::framework;  // NOLINT

118
  BindException(&m);
Y
Yu Yang 已提交
119

S
sneaxiy 已提交
120
  m.def(
S
sneaxiy 已提交
121
      "_append_python_callable_object_and_return_id",
S
sneaxiy 已提交
122 123 124 125
      [](py::object py_obj) -> size_t {
        return paddle::operators::AppendPythonCallableObjectAndReturnId(py_obj);
      });

S
sneaxiy 已提交
126 127 128
  m.add_object("_cleanup",
               py::capsule([]() { ScopePool::Instance().Clear(); }));

M
minqiyang 已提交
129
  py::class_<imperative::VarBase>(m, "VarBase", R"DOC()DOC")
130 131
      // .def(py::init<>())
      .def(py::init<bool>(), py::arg("stop_gradient") = false)
132
      .def("_run_backward",
X
Xin Pan 已提交
133
           [](imperative::VarBase &self) { self.RunBackward(); })
M
minqiyang 已提交
134
      .def("_grad_name", &imperative::VarBase::GradName)
M
minqiyang 已提交
135
      .def("_grad_value", &imperative::VarBase::GradValue)
M
minqiyang 已提交
136
      .def("_grad_ivar",
M
minqiyang 已提交
137
           [](const imperative::VarBase &self) { return self.grads_; },
M
minqiyang 已提交
138
           py::return_value_policy::reference)
M
minqiyang 已提交
139
      .def("value", [](const imperative::VarBase &self) { return self.var_; },
M
minqiyang 已提交
140
           py::return_value_policy::reference)
141 142 143 144 145 146
      .def_property(
          "desc",
          [](const imperative::VarBase &self) { return self.var_desc_; },
          [](imperative::VarBase &self, framework::VarDesc *var_desc) {
            self.var_desc_ = var_desc;
          },
147 148 149 150 151 152
          py::return_value_policy::reference)
      .def_property(
          "stop_gradient",
          [](const imperative::VarBase &self) { return self.stop_gradient_; },
          [](imperative::VarBase &self, bool stop_gradient) {
            self.stop_gradient_ = stop_gradient;
153
          });
154

155
  py::class_<imperative::OpBase, PyOpBase>(m, "OpBase", R"DOC()DOC")
156 157 158 159 160 161 162 163
      .def(py::init<>())
      .def_property(
          "desc", [](const imperative::OpBase &self) { return self.op_desc_; },
          [](imperative::OpBase &self, framework::OpDesc *op_desc) {
            if (op_desc) {
              self.op_desc_ = op_desc;
            }
          },
X
Xin Pan 已提交
164 165 166 167 168 169 170
          py::return_value_policy::reference)
      .def_property(
          "forward_id",
          [](const imperative::OpBase &self) { return self.forward_id_; },
          [](imperative::OpBase &self, int forward_id) {
            self.forward_id_ = forward_id;
          },
X
Xin Pan 已提交
171 172 173 174 175 176 177
          py::return_value_policy::reference)
      .def_property(
          "backward_id",
          [](const imperative::OpBase &self) { return self.backward_id_; },
          [](imperative::OpBase &self, int backward_id) {
            self.backward_id_ = backward_id;
          },
178 179
          py::return_value_policy::reference);

X
Xin Pan 已提交
180
  py::class_<imperative::Layer, Layer /* <--- trampoline*/> layer(m, "Layer");
181
  layer.def(py::init<>())
X
Xin Pan 已提交
182 183 184
      .def("forward", [](imperative::Layer &self,
                         const std::vector<imperative::VarBase> &inputs) {
        return self.Forward(inputs);
X
Xin Pan 已提交
185
      });
X
Xin Pan 已提交
186

X
polish  
Xin Pan 已提交
187
  py::class_<imperative::PyLayer>(m, "PyLayer")
X
Xin Pan 已提交
188
      .def(py::init<>())
X
Xin Pan 已提交
189 190
      .def_static(
          "apply",
X
Xin Pan 已提交
191
          [](int func_id, const std::vector<imperative::VarBase *> &inputs)
X
Xin Pan 已提交
192 193 194 195
              -> std::vector<imperative::VarBase *> {
                return imperative::PyLayer::Apply(func_id, inputs);
              },
          py::return_value_policy::take_ownership)
X
polish  
Xin Pan 已提交
196 197 198 199 200
      .def_static("register_func",
                  [](int func_id, const py::object &callable) {
                    imperative::PyLayer::RegisterFunc(func_id, callable);
                  })
      .def_static("num_funcs", &imperative::PyLayer::NumFuncs);
X
Xin Pan 已提交
201

202 203
  BindTracer(&m);

204 205 206
  py::class_<Tensor>(m, "Tensor", py::buffer_protocol())
      .def_buffer(
          [](Tensor &self) -> py::buffer_info { return CastToPyBuffer(self); })
Y
yuyang18 已提交
207
      .def("_get_dims",
208
           [](const Tensor &self) { return vectorize(self.dims()); })
Y
yuyang18 已提交
209
      .def("_set_dims",
Q
qijun 已提交
210
           [](Tensor &self, const std::vector<int64_t> &dim) {
Y
Yu Yang 已提交
211
             self.Resize(make_ddim(dim));
Y
Yu Yang 已提交
212
           })
Y
yuyang18 已提交
213
      .def("_set_layout",
D
dzhwinter 已提交
214 215 216
           [](Tensor &self, const std::string &layout) {
             self.set_layout(StringToDataLayout(layout));
           })
Y
yuyang18 已提交
217
      .def("_alloc_float",
D
dzhwinter 已提交
218
           [](Tensor &self, paddle::platform::CUDAPlace &place) {
Q
qijun 已提交
219
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
220
           })
Y
yuyang18 已提交
221
      .def("_alloc_float",
Y
Yu Yang 已提交
222
           [](Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
223
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
224
           })
Y
yuyang18 已提交
225
      .def("_alloc_int",
Y
Yu Yang 已提交
226
           [](Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
227
             self.mutable_data<int>(place);
Y
Yu Yang 已提交
228
           })
Y
yuyang18 已提交
229
      .def("_alloc_int",
D
dzhwinter 已提交
230
           [](Tensor &self, paddle::platform::CUDAPlace &place) {
Q
qijun 已提交
231
             self.mutable_data<int>(place);
Q
qijun 已提交
232
           })
Y
yuyang18 已提交
233
      .def("_alloc_int",
C
chengduoZH 已提交
234 235 236
           [](Tensor &self, paddle::platform::CUDAPinnedPlace &place) {
             self.mutable_data<int>(place);
           })
Y
yuyang18 已提交
237
      .def("_alloc_float",
C
chengduoZH 已提交
238 239 240
           [](Tensor &self, paddle::platform::CUDAPinnedPlace &place) {
             self.mutable_data<float>(place);
           })
Y
Yu Yang 已提交
241 242
      .def("set", PyCPUTensorSetFromArray<float>)
      .def("set", PyCPUTensorSetFromArray<int>)
243
      .def("set", PyCPUTensorSetFromArray<double>)
244
      .def("set", PyCPUTensorSetFromArray<int64_t>)
Y
Yu Yang 已提交
245
      .def("set", PyCPUTensorSetFromArray<bool>)
246
      .def("set", PyCPUTensorSetFromArray<uint16_t>)
F
fengjiayi 已提交
247
      .def("set", PyCPUTensorSetFromArray<uint8_t>)
Q
qingqing01 已提交
248
      .def("set", PyCPUTensorSetFromArray<int8_t>)
249
#ifdef PADDLE_WITH_CUDA
Y
Yu Yang 已提交
250 251
      .def("set", PyCUDATensorSetFromArray<float>)
      .def("set", PyCUDATensorSetFromArray<int>)
252
      .def("set", PyCUDATensorSetFromArray<double>)
253
      .def("set", PyCUDATensorSetFromArray<int64_t>)
Y
Yu Yang 已提交
254
      .def("set", PyCUDATensorSetFromArray<bool>)
255
      .def("set", PyCUDATensorSetFromArray<uint16_t>)
F
fengjiayi 已提交
256
      .def("set", PyCUDATensorSetFromArray<uint8_t>)
Q
qingqing01 已提交
257
      .def("set", PyCUDATensorSetFromArray<int8_t>)
C
chengduoZH 已提交
258 259 260 261 262 263
      .def("set", PyCUDAPinnedTensorSetFromArray<float>)
      .def("set", PyCUDAPinnedTensorSetFromArray<int>)
      .def("set", PyCUDAPinnedTensorSetFromArray<double>)
      .def("set", PyCUDAPinnedTensorSetFromArray<int64_t>)
      .def("set", PyCUDAPinnedTensorSetFromArray<bool>)
      .def("set", PyCUDAPinnedTensorSetFromArray<uint16_t>)
F
fengjiayi 已提交
264
      .def("set", PyCUDAPinnedTensorSetFromArray<uint8_t>)
Q
qingqing01 已提交
265
      .def("set", PyCUDAPinnedTensorSetFromArray<int8_t>)
Q
qijun 已提交
266
#endif
267
      .def("shape", [](Tensor &self) { return vectorize(self.dims()); })
Y
yuyang18 已提交
268 269 270 271
      .def("_set_float_element", TensorSetElement<float>)
      .def("_get_float_element", TensorGetElement<float>)
      .def("_set_double_element", TensorSetElement<double>)
      .def("_get_double_element", TensorGetElement<double>)
Y
Yu Yang 已提交
272
      .def("_dtype", [](Tensor &self) { return self.type(); });
Y
Yu Yang 已提交
273

X
Xin Pan 已提交
274 275 276 277 278 279 280 281 282 283 284 285 286
  py::class_<LoDTensor, Tensor>(m, "LoDTensor", R"DOC(
    LoDTensor is a Tensor with optional LoD information.

    np.array(lod_tensor) can convert LoDTensor to numpy array.
    lod_tensor.lod() can retrieve the LoD information.

    LoD is short for Level of Details and is usually used for varied sequence
    length. You can skip the following comment if you don't need optional LoD.

  For example:
     A LoDTensor X can look like the example below. It contains 2 sequences.
     The first has length 2 and the second has length 3, as described by x.lod.

X
fix doc  
Xin Pan 已提交
287
     The first tensor dimension 5=2+3 is calculated from LoD if it's available.
X
Xin Pan 已提交
288
     It means the total number of sequence element. In X, each element has 2
X
fix doc  
Xin Pan 已提交
289
     columns, hence [5, 2].
X
Xin Pan 已提交
290 291 292

      x.lod  = [[2, 3]]
      x.data = [[1, 2], [3, 4],
X
fix doc  
Xin Pan 已提交
293 294
                [5, 6], [7, 8], [9, 10]]
      x.shape = [5, 2]
X
Xin Pan 已提交
295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317

      LoD can have multiple levels (for example, a paragraph can have multiple
      sentences and a sentence can have multiple words). In the following
      LodTensor Y, the lod_level is 2. It means there are 2 sequence, the
      first sequence length is 2 (has 2 sub-sequences), the second one's
      length is 1. The first sequence's 2 sub-sequences have length 2 and 2,
      respectively. And the second sequence's 1 sub-sequence has length 3.

      y.lod = [[2 1], [2 2 3]]
      y.shape = [2+2+3, ...]

  Note:
      In above description, LoD is length-based. In Paddle internal
      implementation, lod is offset-based. Hence, internally,
      y.lod is represented as [[0, 2, 3], [0, 2, 4, 7]] (length-based
      equivlent would be [[2-0, 3-2], [2-0, 4-2, 7-4]]).

      Sometimes LoD is called recursive_sequence_length to be more
      self-explanatory. In this case, it must be length-based. Due to history
      reasons. when LoD is called lod in public API, it might be offset-based.
      Users should be careful about it.

        )DOC")
318 319
      .def_buffer(
          [](Tensor &self) -> py::buffer_info { return CastToPyBuffer(self); })
320 321 322 323 324 325 326 327 328 329 330 331 332 333
      .def("__init__",
           [](LoDTensor &instance, const std::vector<std::vector<size_t>>
                                       &recursive_sequence_lengths) {
             LoD new_lod;
             new_lod.reserve(recursive_sequence_lengths.size());
             std::copy(recursive_sequence_lengths.begin(),
                       recursive_sequence_lengths.end(),
                       std::back_inserter(new_lod));
             LoD new_offset_lod = ConvertToOffsetBasedLoD(new_lod);
             PADDLE_ENFORCE(
                 CheckLoD(new_offset_lod, -1),
                 "the provided recursive_sequence_lengths info is invalid");
             new (&instance) LoDTensor(new_offset_lod);
           })
Y
Yu Yang 已提交
334
      .def("__init__", [](LoDTensor &instance) { new (&instance) LoDTensor(); })
G
gongweibao 已提交
335 336 337 338 339
      // We implement offset based LOD in C++ while we use length based with
      // Python API. So we changed set_lod to set_recursive_sequence_lengths to
      // avoid misuse.
      // The discussion is here:
      // https://github.com/PaddlePaddle/Paddle/issues/10855
D
dangqingqing 已提交
340
      .def("set_lod",
341
           [](LoDTensor &self, const std::vector<std::vector<size_t>> &lod) {
342
             // the input lod is offset-based level-of-detail info
Y
Yu Yang 已提交
343
             LoD new_lod;
344 345
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
346 347
             PADDLE_ENFORCE(CheckLoD(new_lod, vectorize(self.dims()).front()),
                            "the provided lod info is invalid");
348
             self.set_lod(new_lod);
D
dangqingqing 已提交
349
           })
350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374
      .def("set_recursive_sequence_lengths",
           [](LoDTensor &self, const std::vector<std::vector<size_t>>
                                   &recursive_sequence_lengths) {
             // the input recursive_sequence_lengths is length-based
             // level-of-detail info
             LoD new_lod;
             new_lod.reserve(recursive_sequence_lengths.size());
             std::copy(recursive_sequence_lengths.begin(),
                       recursive_sequence_lengths.end(),
                       std::back_inserter(new_lod));
             LoD new_offset_lod = ConvertToOffsetBasedLoD(new_lod);
             PADDLE_ENFORCE(
                 CheckLoD(new_offset_lod, vectorize(self.dims()).front()),
                 "the provided recursive_sequence_lengths info is invalid");
             self.set_lod(new_offset_lod);
           })
      .def("lod",
           [](LoDTensor &self) -> std::vector<std::vector<size_t>> {
             // output the offset-based lod info
             LoD lod = self.lod();
             std::vector<std::vector<size_t>> new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             return new_lod;
           })
G
gongweibao 已提交
375
      // Set above comments of set_lod.
376 377 378 379 380 381 382 383 384 385 386 387 388
      .def("recursive_sequence_lengths",
           [](LoDTensor &self) -> std::vector<std::vector<size_t>> {
             // output the length-based lod info
             LoD lod = ConvertToLengthBasedLoD(self.lod());
             std::vector<std::vector<size_t>> new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             return new_lod;
           })
      .def("has_valid_recursive_sequence_lengths", [](LoDTensor &self) -> bool {
        // Check that the lod info is valid and match the outermost
        // dimension of the LoDTensor data
        return CheckLoD(self.lod(), vectorize(self.dims()).front());
D
dangqingqing 已提交
389 390
      });

Q
qijun 已提交
391 392 393 394 395 396 397 398 399 400 401
  py::class_<SelectedRows>(m, "SelectedRows")
      .def("__init__",
           [](SelectedRows &instance) { new (&instance) SelectedRows(); })
      .def("__init__",
           [](SelectedRows &instance, const std::vector<int64_t> rows,
              const int64_t &height) {
             new (&instance) SelectedRows(rows, height);
           })
      .def("get_tensor",
           [](SelectedRows &self) { return self.mutable_value(); },
           py::return_value_policy::reference)
402 403
      .def("numel",
           [](SelectedRows &self) -> int64_t { return self.value().numel(); })
Q
qijun 已提交
404 405
      .def("set_height", &SelectedRows::set_height)
      .def("height", &SelectedRows::height)
Q
qijun 已提交
406 407 408 409 410 411 412 413 414
      .def("set_rows",
           [](SelectedRows &self, std::vector<int64_t> rows) {
#ifndef PADDLE_WITH_CUDA
             self.set_rows(rows);
#else
        Vector<int64_t> new_rows(rows);
        self.set_rows(new_rows);
#endif
           })
415
      .def("sync_index", [](SelectedRows &instance) { instance.SyncIndex(); })
416
      .def("rows", [](SelectedRows &self) {
417 418 419 420 421
        auto rows = self.rows();
        std::vector<int64_t> new_rows;
        new_rows.reserve(rows.size());
        std::copy(rows.begin(), rows.end(), std::back_inserter(new_rows));
        return new_rows;
422
      });
Q
qijun 已提交
423

424
  py::class_<Variable>(m, "Variable", R"DOC(Variable Class.
425 426 427

All parameter, weight, gradient are variables in Paddle.
)DOC")
428
      .def("is_int", [](const Variable &var) { return var.IsType<int>(); })
429
      .def("set_int",
430 431
           [](Variable &var, int val) -> void { *var.GetMutable<int>() = val; })
      .def("get_int", [](const Variable &var) -> int { return var.Get<int>(); })
432 433 434 435 436 437 438
      .def("is_float", [](const Variable &var) { return var.IsType<float>(); })
      .def("set_float",
           [](Variable &var, float val) -> void {
             *var.GetMutable<float>() = val;
           })
      .def("get_float",
           [](const Variable &var) -> float { return var.Get<float>(); })
Y
Yu Yang 已提交
439
      .def("get_tensor",
440 441
           [](Variable &self) -> LoDTensor * {
             return self.GetMutable<LoDTensor>();
D
dangqingqing 已提交
442 443
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
444 445 446
      .def("get_lod_rank_table",
           [](Variable &self) { return self.GetMutable<LoDRankTable>(); },
           py::return_value_policy::reference)
Q
qijun 已提交
447 448 449 450 451
      .def("get_selected_rows",
           [](Variable &self) -> SelectedRows * {
             return self.GetMutable<SelectedRows>();
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
452 453 454
      .def("get_lod_tensor_array",
           [](Variable &self) { return self.GetMutable<LoDTensorArray>(); },
           py::return_value_policy::reference)
P
peizhilin 已提交
455
#if (defined(PADDLE_WITH_CUDA) && !defined(_WIN32))
D
Dong Zhihong 已提交
456 457 458 459 460
      .def("get_communicator",
           [](Variable &self) -> platform::Communicator * {
             return self.GetMutable<platform::Communicator>();
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
461
#endif
Y
Refine  
Yu Yang 已提交
462 463 464 465 466
      .def("get_reader",
           [](Variable &self) -> framework::ReaderHolder * {
             PADDLE_ENFORCE(self.IsType<framework::ReaderHolder>());
             return self.GetMutable<framework::ReaderHolder>();
           },
W
wopeizl 已提交
467
           py::return_value_policy::reference);
468

Y
Refine  
Yu Yang 已提交
469
  py::class_<framework::ReaderHolder>(m, "Reader", "")
470
      .def("reset", &framework::ReaderHolder::ResetAll);
Y
Refine  
Yu Yang 已提交
471

S
sneaxiy 已提交
472 473 474 475
  using LoDTensorBlockingQueue =
      ::paddle::operators::reader::LoDTensorBlockingQueue;
  using LoDTensorBlockingQueueHolder =
      ::paddle::operators::reader::LoDTensorBlockingQueueHolder;
S
sneaxiy 已提交
476 477
  py::class_<LoDTensorBlockingQueue, std::shared_ptr<LoDTensorBlockingQueue>>(
      m, "LoDTensorBlockingQueue", "")
S
sneaxiy 已提交
478
      .def("push",
S
sneaxiy 已提交
479
           [](LoDTensorBlockingQueue &self,
S
sneaxiy 已提交
480
              const std::vector<framework::LoDTensor> &lod_tensor_vec) {
S
sneaxiy 已提交
481
             pybind11::gil_scoped_release release;
S
sneaxiy 已提交
482
             return self.Push(lod_tensor_vec);
S
sneaxiy 已提交
483
           })
S
sneaxiy 已提交
484 485 486 487
      .def("size", &LoDTensorBlockingQueue::Size)
      .def("capacity", &LoDTensorBlockingQueue::Cap)
      .def("close", &LoDTensorBlockingQueue::Close)
      .def("is_closed", &LoDTensorBlockingQueue::IsClosed);
S
sneaxiy 已提交
488

S
sneaxiy 已提交
489
  m.def("init_lod_tensor_blocking_queue",
S
sneaxiy 已提交
490
        [](Variable &var, size_t capacity,
S
sneaxiy 已提交
491
           const std::vector<std::vector<int64_t>> &shapes)
S
sneaxiy 已提交
492
            -> std::shared_ptr<LoDTensorBlockingQueue> {
S
sneaxiy 已提交
493 494 495 496 497 498
              std::vector<DDim> dims(shapes.size());
              std::transform(shapes.begin(), shapes.end(), dims.begin(),
                             [](const std::vector<int64_t> &shape) {
                               return make_ddim(shape);
                             });
              auto *holder = var.GetMutable<LoDTensorBlockingQueueHolder>();
499 500
              holder->InitOnce(capacity, dims,
                               FLAGS_reader_queue_speed_test_mode);
S
sneaxiy 已提交
501
              return holder->GetQueue();
S
sneaxiy 已提交
502
            },
S
sneaxiy 已提交
503
        py::return_value_policy::copy);
S
sneaxiy 已提交
504

S
sneaxiy 已提交
505
  py::class_<Scope>(m, "_Scope", R"DOC(
Q
Qiao Longfei 已提交
506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524
    Scope is an association of a name to Variable. All variables belong to Scope.

    Variables in a parent scope can be retrieved from local scope.

    You need to specify a scope to run a Net, i.e., `exe.Run(&scope)`.
    One net can run in different scopes and update different variable in the
    scope.

    You can create var in a scope and get it from the scope.

    Examples:
        .. code-block:: python

          # create tensor from a scope and set value to it.
          param = scope.var('Param').get_tensor()
          param_array = np.full((height, row_numel), 5.0).astype("float32")
          param.set(param_array, place)

        )DOC")
S
sneaxiy 已提交
525 526
      .def("_remove_from_pool",
           [](Scope &self) { ScopePool::Instance().Remove(&self); })
D
dongzhihong 已提交
527
      .def("var",
528
           [](Scope &self, const std::string &name) -> Variable * {
D
dongzhihong 已提交
529
             return self.Var(name);
Y
Yu Yang 已提交
530
           },
531
           py::return_value_policy::reference)
532
      .def("find_var", &Scope::FindVar, py::return_value_policy::reference)
533
      .def("new_scope", [](Scope &self) -> Scope * { return &self.NewScope(); },
534
           py::return_value_policy::reference)
Y
Yu Yang 已提交
535
      .def("drop_kids", &Scope::DropKids);
536

S
sneaxiy 已提交
537 538 539 540 541 542 543 544
  m.def("Scope",
        []() -> Scope * {
          auto *s = new Scope();
          ScopePool::Instance().Insert(std::unique_ptr<Scope>(s));
          return s;
        },
        py::return_value_policy::reference);

Y
Yu Yang 已提交
545 546
  //! @note: Be careful! PyBind will return std::string as an unicode, not
  //! Python str. If you want a str object, you should cast them in Python.
Y
Yu Yang 已提交
547 548
  m.def("get_all_op_protos", []() -> std::vector<py::bytes> {
    std::vector<py::bytes> ret_values;
549 550 551 552 553 554 555 556 557 558
    for (auto &iter : OpInfoMap::Instance().map()) {
      auto &info = iter.second;
      if (info.HasOpProtoAndChecker()) {
        std::string str;
        PADDLE_ENFORCE(
            info.Proto().SerializeToString(&str),
            "Serialize OpProto Error. This could be a bug of Paddle.");
        ret_values.emplace_back(str);
      }
    }
Y
Yu Yang 已提交
559 560
    return ret_values;
  });
561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576
  m.def(
      "get_grad_op_desc", [](const OpDesc &op_desc,
                             const std::unordered_set<std::string> &no_grad_set,
                             const std::vector<BlockDesc *> &grad_sub_block) {
        std::unordered_map<std::string, std::string> grad_to_var;
        std::vector<std::unique_ptr<OpDesc>> grad_op_descs =
            framework::OpInfoMap::Instance()
                .Get(op_desc.Type())
                .GradOpMaker()(op_desc, no_grad_set, &grad_to_var,
                               grad_sub_block);
        std::vector<OpDesc *> grad_op_desc_ptrs(grad_op_descs.size());
        std::transform(grad_op_descs.begin(), grad_op_descs.end(),
                       grad_op_desc_ptrs.begin(),
                       [](std::unique_ptr<OpDesc> &p) { return p.release(); });
        return std::make_pair(grad_op_desc_ptrs, grad_to_var);
      });
Y
Yu Yang 已提交
577
  m.def("prune", [](const ProgramDesc &origin,
578
                    const std::vector<std::array<size_t, 2>> &targets) {
Y
Yu Yang 已提交
579
    ProgramDesc prog_with_targets(origin);
580
    for (const auto &t : targets) {
581
      prog_with_targets.MutableBlock(t[0])->Op(t[1])->SetIsTarget(true);
582
    }
583
    proto::ProgramDesc pruned_desc;
584
    Prune(*prog_with_targets.Proto(), &pruned_desc);
Y
Yu Yang 已提交
585
    return new ProgramDesc(pruned_desc);
586
  });
587 588 589 590
  m.def("empty_var_name",
        []() { return std::string(framework::kEmptyVarName); });
  m.def("grad_var_suffix",
        []() { return std::string(framework::kGradVarSuffix); });
591 592 593
  m.def_submodule(
       "var_names",
       "The module will return special predefined variable name in Paddle")
Y
Yi Wang 已提交
594 595
      .def("empty", []() { return kEmptyVarName; })
      .def("temp", []() { return kTempVarName; });
Q
qijun 已提交
596
  // clang-format off
Y
Yu Yang 已提交
597
  py::class_<paddle::platform::DeviceContext>(m, "DeviceContext")
Q
qijun 已提交
598 599
      .def_static("create",
                  [](paddle::platform::CPUPlace& place)
Q
qijun 已提交
600
                      -> paddle::platform::DeviceContext* {
Q
qijun 已提交
601 602 603
                    return new paddle::platform::CPUDeviceContext();
                  })
      .def_static("create",
D
dzhwinter 已提交
604
                  [](paddle::platform::CUDAPlace& place)
Q
qijun 已提交
605
                      -> paddle::platform::DeviceContext* {
606
#ifndef PADDLE_WITH_CUDA
D
dzhwinter 已提交
607
                    PADDLE_THROW("CUDAPlace is not supported in CPU device.");
Q
qijun 已提交
608
#else
Q
qijun 已提交
609
                    return new paddle::platform::CUDADeviceContext(place);
Q
qijun 已提交
610
#endif
C
chengduoZH 已提交
611 612 613 614 615 616 617 618 619 620 621
                  })
          .def_static("create",
                [](paddle::platform::CUDAPinnedPlace& place)
                        -> paddle::platform::DeviceContext* {
#ifndef PADDLE_WITH_CUDA
                  PADDLE_THROW(
                        "CUDAPinnedPlace is not supported in CPU device.");
#else
                  return new paddle::platform::CUDAPinnedDeviceContext(place);
#endif
                });;
D
Dong Zhihong 已提交
622
// clang-format on
P
peizhilin 已提交
623
#if (defined(PADDLE_WITH_CUDA) && !defined(_WIN32))
D
Dong Zhihong 已提交
624 625
  py::class_<platform::Communicator>(m, "Communicator").def(py::init<>());
#endif
D
dzhwinter 已提交
626
  py::class_<platform::CUDAPlace>(m, "CUDAPlace")
627
      .def(py::init<int>())
D
dzhwinter 已提交
628
      .def("__str__", string::to_string<const platform::CUDAPlace &>);
Q
qijun 已提交
629

630 631 632
  py::class_<paddle::platform::CPUPlace>(m, "CPUPlace")
      .def(py::init<>())
      .def("__str__", string::to_string<const platform::CPUPlace &>);
Y
Yu Yang 已提交
633

C
chengduoZH 已提交
634 635 636 637
  py::class_<paddle::platform::CUDAPinnedPlace>(m, "CUDAPinnedPlace")
      .def(py::init<>())
      .def("__str__", string::to_string<const platform::CUDAPinnedPlace &>);

Y
Yu Yang 已提交
638 639 640 641 642 643 644
  py::class_<platform::Place>(m, "Place")
      .def(py::init<>())
      .def("set_place",
           [](platform::Place &self, const platform::CPUPlace &cpu_place) {
             self = cpu_place;
           })
      .def("set_place",
D
dzhwinter 已提交
645
           [](platform::Place &self, const platform::CUDAPlace &gpu_place) {
Y
Yu Yang 已提交
646
             self = gpu_place;
C
chengduoZH 已提交
647 648
           })
      .def("set_place", [](platform::Place &self,
C
chengduoZH 已提交
649 650
                           const platform::CUDAPinnedPlace &cuda_pinned_place) {
        self = cuda_pinned_place;
C
chengduoZH 已提交
651
      });
Y
Yu Yang 已提交
652

Y
Yu Yang 已提交
653 654 655
  py::class_<OperatorBase>(m, "Operator")
      .def_static("create",
                  [](py::bytes protobin) {
656
                    proto::OpDesc desc;
Y
Yu Yang 已提交
657 658 659 660 661
                    PADDLE_ENFORCE(desc.ParsePartialFromString(protobin),
                                   "Cannot parse user input to OpDesc");
                    PADDLE_ENFORCE(desc.IsInitialized(),
                                   "User OpDesc is not initialized, reason %s",
                                   desc.InitializationErrorString());
662
                    return OpRegistry::CreateOp(desc);
Y
Yu Yang 已提交
663
                  })
664
      .def("run",
665
           [](OperatorBase &self, const Scope &scope,
D
dzhwinter 已提交
666 667 668
              const platform::CPUPlace &place) { self.Run(scope, place); })
      .def("run",
           [](OperatorBase &self, const Scope &scope,
D
dzhwinter 已提交
669
              const platform::CUDAPlace &place) { self.Run(scope, place); })
C
chengduoZH 已提交
670 671 672 673 674
      .def("run",
           [](OperatorBase &self, const Scope &scope,
              const platform::CUDAPinnedPlace &place) {
             self.Run(scope, place);
           })
Y
Yu Yang 已提交
675 676 677 678 679 680 681
      .def("type",
           [](const OperatorBase &op) -> std::string { return op.Type(); })
      .def("outputs",
           [](const OperatorBase &op)
               -> std::map<std::string, std::vector<std::string>> {
                 return op.Outputs();
               })
Q
qijun 已提交
682 683
      .def("output_vars",
           [](const OperatorBase &op) { return op.OutputVars(true); })
Y
Yu Yang 已提交
684
      .def("inputs", [](const OperatorBase &op) { return op.Inputs(); })
Q
qijun 已提交
685
      .def("input_vars", [](const OperatorBase &op) { return op.InputVars(); })
Y
Yu Yang 已提交
686 687 688 689
      .def("__str__", &OperatorBase::DebugString)
      .def("no_intermediate_outputs",
           [](const OperatorBase &op) { return op.OutputVars(false); })
      .def("support_gpu", &OperatorBase::SupportGPU);
Y
Yu Yang 已提交
690

F
fengjiayi 已提交
691
  py::class_<framework::Executor>(m, "Executor")
D
dzhwinter 已提交
692
      .def(py::init<const platform::Place &>())
Y
Yancey1989 已提交
693
      .def("close", &Executor::Close)
S
sneaxiy 已提交
694 695 696 697 698
      .def("run", [](Executor &self, const ProgramDesc &prog, Scope *scope,
                     int block_id, bool create_local_scope, bool create_vars) {
        pybind11::gil_scoped_release release;
        self.Run(prog, scope, block_id, create_local_scope, create_vars);
      });
S
sneaxiy 已提交
699

D
dzhwinter 已提交
700
  m.def("init_gflags", framework::InitGflags);
Y
Yang Yu 已提交
701
  m.def("init_glog", framework::InitGLOG);
X
Xin Pan 已提交
702 703
  m.def("init_devices",
        [](bool init_p2p) { framework::InitDevices(init_p2p); });
704

705
  m.def("is_compiled_with_cuda", IsCompiledWithCUDA);
706
  m.def("is_compiled_with_brpc", IsCompiledWithBrpc);
Y
update  
Yancey1989 已提交
707
  m.def("is_compiled_with_dist", IsCompiledWithDIST);
708 709 710 711 712 713
#ifdef PADDLE_WITH_CUDA
  m.def("is_float16_supported", [](const platform::CUDAPlace &place) -> bool {
    // Only GPUs with Compute Capability >= 53 support float16
    return platform::GetCUDAComputeCapability(place.device) >= 53;
  });
#endif
714

715
  m.def("set_feed_variable", framework::SetFeedVariable);
Q
qijun 已提交
716
  m.def("get_fetch_variable", framework::GetFetchVariable);
717
  m.def("get_variable_tensor", framework::GetVariableTensor);
Q
qijun 已提交
718

X
Xin Pan 已提交
719 720
  m.def("_is_program_version_supported", IsProgramVersionSupported);

721 722 723 724 725
  BindProgramDesc(&m);
  BindBlockDesc(&m);
  BindVarDsec(&m);
  BindOpDesc(&m);
  BindConstValue(&m);
Y
Yu Yang 已提交
726

Y
Yu Yang 已提交
727 728 729 730 731 732 733 734 735
  py::class_<framework::LoDRankTable>(m, "LodRankTable")
      .def("items", [](framework::LoDRankTable &table) {
        std::vector<std::pair<size_t, size_t>> res;
        for (auto &item : table.items()) {
          res.push_back({item.index, item.length});
        }
        return res;
      });

Y
Yu Yang 已提交
736
  py::class_<LoDTensorArray>(m, "LoDTensorArray")
S
sneaxiy 已提交
737 738
      .def("__init__",
           [](LoDTensorArray &instance) { new (&instance) LoDTensorArray(); })
Y
Yu Yang 已提交
739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754
      .def("__getitem__",
           [](LoDTensorArray &self, size_t i) { return &self.at(i); },
           py::return_value_policy::reference)
      .def("__len__", [](LoDTensorArray &self) { return self.size(); })
      .def("__setitem__",
           [](LoDTensorArray &self, size_t i, const LoDTensor &t) {
             PADDLE_ENFORCE_LT(i, self.size());
             self[i].ShareDataWith(t);
             self[i].set_lod(t.lod());
           })
      .def("append", [](LoDTensorArray &self, const LoDTensor &t) {
        self.emplace_back();
        self.back().ShareDataWith(t);
        self.back().set_lod(t.lod());
      });

D
dzhwinter 已提交
755 756 757
  m.def("IsInplace",
        [](std::string op) -> bool { return operators::IsInplace(op); });

Y
Yu Yang 已提交
758
  m.def("op_support_gpu", OpSupportGPU);
D
Dong Zhihong 已提交
759
#ifdef PADDLE_WITH_CUDA
D
Dong Zhihong 已提交
760
  m.def("get_cuda_device_count", platform::GetCUDADeviceCount);
D
dangqingqing 已提交
761

P
peizhilin 已提交
762
#ifndef _WIN32
D
dangqingqing 已提交
763 764 765
  m.def("nvprof_init", platform::CudaProfilerInit);
  m.def("nvprof_start", platform::CudaProfilerStart);
  m.def("nvprof_stop", platform::CudaProfilerStop);
D
Dong Zhihong 已提交
766
#endif
P
peizhilin 已提交
767
#endif
Y
Yu Yang 已提交
768

769 770 771 772
  py::enum_<platform::ProfilerState>(m, "ProfilerState", py::arithmetic())
      .value("kDisabled", platform::ProfilerState::kDisabled)
      .value("kCPU", platform::ProfilerState::kCPU)
      .value("kCUDA", platform::ProfilerState::kCUDA)
773
      .value("kAll", platform::ProfilerState::kAll)
774 775 776 777 778 779 780 781 782 783 784 785 786
      .export_values();

  py::enum_<platform::EventSortingKey>(m, "EventSortingKey", py::arithmetic())
      .value("kDefault", platform::EventSortingKey::kDefault)
      .value("kCalls", platform::EventSortingKey::kCalls)
      .value("kTotal", platform::EventSortingKey::kTotal)
      .value("kMin", platform::EventSortingKey::kMin)
      .value("kMax", platform::EventSortingKey::kMax)
      .value("kAve", platform::EventSortingKey::kAve)
      .export_values();

  m.def("enable_profiler", platform::EnableProfiler);
  m.def("disable_profiler", platform::DisableProfiler);
X
Xin Pan 已提交
787
  m.def("is_profiler_enabled", platform::IsProfileEnabled);
788
  m.def("reset_profiler", platform::ResetProfiler);
Y
Yu Yang 已提交
789

790 791
  py::class_<ir::Pass, std::shared_ptr<ir::Pass>> pass(m, "Pass");
  pass.def(py::init())
792 793 794 795 796
      .def(
          "set_str",
          [](ir::Pass &self, const std::string &name, const std::string &attr) {
            self.Set<std::string>(name, new std::string(attr));
          })
X
Xin Pan 已提交
797 798
      .def("set_int", [](ir::Pass &self, const std::string &name,
                         int val) { self.Set<const int>(name, new int(val)); })
F
flame 已提交
799 800 801 802 803 804
      .def("type", &ir::Pass::Type)
      .def("apply", [](ir::Pass &self, std::shared_ptr<ir::Graph> graph) {
        std::unique_ptr<ir::Graph> origin_graph(graph.get());
        auto optim_graph = self.Apply(std::move(origin_graph));
        graph.reset(optim_graph.release());
      });
805

X
fix  
Xin Pan 已提交
806 807
  py::class_<ir::PassBuilder, std::shared_ptr<ir::PassBuilder>> pb(
      m, "PassBuilder");
808 809 810 811 812 813 814 815 816 817 818 819 820 821
  pb.def(py::init())
      .def("append_pass",
           [](ir::PassBuilder &self,
              const std::string &pass_type) -> std::shared_ptr<ir::Pass> {
             return self.AppendPass(pass_type);
           })
      .def("all_passes", [](ir::PassBuilder &self) { return self.AllPasses(); })
      .def("insert_pass",
           [](ir::PassBuilder &self, size_t idx, const std::string &pass_type) {
             return self.InsertPass(idx, pass_type);
           })
      .def("remove_pass",
           [](ir::PassBuilder &self, size_t idx) { self.RemovePass(idx); });

Y
yuyang18 已提交
822
  // -- python binds for parallel executor.
Y
yuyang18 已提交
823
  py::class_<ParallelExecutor> pe(m, "ParallelExecutor");
C
chengduo 已提交
824 825 826 827
  py::class_<ExecutionStrategy> exec_strategy(pe, "ExecutionStrategy", R"DOC(
    ExecutionStrategy allows the user to more preciously control how to run
    the program in ParallelExecutor by setting the property.

C
chengduo 已提交
828 829 830 831 832 833 834 835 836 837 838
    Examples:
        .. code-block:: python

          exec_strategy = fluid.ExecutionStrategy()
          exec_strategy.num_threads = 4

          train_exe = fluid.ParallelExecutor(use_cuda=True,
                                             loss_name=loss.name,
                                             exec_strategy=exec_strategy)

          train_loss, = train_exe.run([loss.name], feed=feed_dict)
C
chengduo 已提交
839 840 841

        )DOC");

Y
yuyang18 已提交
842
  exec_strategy.def(py::init())
Y
yuyang18 已提交
843 844 845 846 847
      .def_property(
          "num_threads",
          [](const ExecutionStrategy &self) { return self.num_threads_; },
          [](ExecutionStrategy &self, size_t num_threads) {
            self.num_threads_ = num_threads;
C
chengduo 已提交
848 849 850 851 852 853 854 855 856 857
          },
          R"DOC(The type is INT, num_threads represents the size of thread pool that
            used to run the operators of the current program in ParallelExecutor.
            If :math:`num\_threads=1`, all the operators will execute one by one,
            but the order maybe difference between iterations.
            If it is not set, it will be set in ParallelExecutor according to the
            device type and device count, for GPU, :math:`num\_threads=device\_count*4`, for CPU,
            :math:`num\_threads=CPU\_NUM*4`, the explanation of:math:`CPU\_NUM` is in ParallelExecutor.
            if it is not set, ParallelExecutor will get the cpu count by calling
            `multiprocessing.cpu_count()`. Default 0.)DOC")
Y
yuyang18 已提交
858
      .def_property(
859 860 861 862
          "use_cuda",
          [](const ExecutionStrategy &self) { return self.use_cuda_; },
          [](ExecutionStrategy &self, bool use_cuda) {
            self.use_cuda_ = use_cuda;
C
chengduo 已提交
863 864 865 866
          })  // FIXME(chengduo): Doesn't add doc for 'use_cuda', use_cuda may
      // make user confuse, because ParallelExecutor has a parameter named
      // 'use_cuda' too, in current implementation, ParallelExecutor's
      // 'use_cuda' will rewrite ExecutionStrategy's 'use_cuda'.
Y
yuyang18 已提交
867 868 869 870 871
      .def_property(
          "allow_op_delay",
          [](const ExecutionStrategy &self) { return self.allow_op_delay_; },
          [](ExecutionStrategy &self, bool allow_op_delay) {
            self.allow_op_delay_ = allow_op_delay;
C
chengduo 已提交
872 873 874 875
          },
          R"DOC(The type is BOOL, allow_op_delay represents whether to delay the
                communication operators to run, it may make the execution faster.
                Note that in some models, allow_op_delay may cause program hang. Default False.)DOC")
Y
yuyang18 已提交
876 877 878 879 880 881 882
      .def_property(
          "num_iteration_per_drop_scope",
          [](const ExecutionStrategy &self) {
            return self.num_iteration_per_drop_scope_;
          },
          [](ExecutionStrategy &self, size_t num_iteration_per_drop_scope) {
            self.num_iteration_per_drop_scope_ = num_iteration_per_drop_scope;
C
chengduo 已提交
883 884 885 886 887 888 889 890 891 892 893
          },
          R"DOC(The type is INT, num_iteration_per_drop_scope indicates how
                many iterations to clean up the temp variables which
                is generated during execution. It may make the execution faster,
                because the temp variable's shape maybe the same between two iterations. Default 100.

                NOTES:
                    1. If you fetch data when calling the 'run', the ParallelExecutor
                       will clean up the temp variables at the end of the current iteration.
                    2. In some NLP model, it may cause the GPU memory is insufficient,
                       in this case, you should reduce `num_iteration_per_drop_scope`.
894 895 896 897 898 899
              )DOC")
      .def_property("_dry_run",
                    [](const ExecutionStrategy &self) { return self.dry_run_; },
                    [](ExecutionStrategy &self, bool dry_run) {
                      self.dry_run_ = dry_run;
                    });
C
chengduo 已提交
900

Y
yuyang18 已提交
901
  exec_strategy.def_property(
Y
yuyang18 已提交
902 903 904 905 906 907 908
      "use_experimental_executor",
      [](const ExecutionStrategy &self) {
        return self.type_ == ExecutionStrategy::kExperimental;
      },
      [](ExecutionStrategy &self, bool experimental) {
        self.type_ = experimental ? ExecutionStrategy::kExperimental
                                  : ExecutionStrategy::kDefault;
Y
yuyang18 已提交
909 910
      });

C
chengduo 已提交
911 912 913 914
  py::class_<BuildStrategy> build_strategy(pe, "BuildStrategy", R"DOC(
    BuildStrategy allows the user to more preciously control how to
    build the SSA Graph in ParallelExecutor by setting the property.

C
chengduo 已提交
915 916 917 918 919 920 921 922 923 924 925
    Examples:
        .. code-block:: python

          build_strategy = fluid.BuildStrategy()
          build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce

          train_exe = fluid.ParallelExecutor(use_cuda=True,
                                             loss_name=loss.name,
                                             build_strategy=build_strategy)

          train_loss, = train_exe.run([loss.name], feed=feed_dict)
C
chengduo 已提交
926
)DOC");
Y
yuyang18 已提交
927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942

  py::enum_<BuildStrategy::ReduceStrategy>(build_strategy, "ReduceStrategy")
      .value("Reduce", BuildStrategy::ReduceStrategy::kReduce)
      .value("AllReduce", BuildStrategy::ReduceStrategy::kAllReduce);
  py::enum_<BuildStrategy::GradientScaleStrategy>(build_strategy,
                                                  "GradientScaleStrategy")
      .value("CoeffNumDevice",
             BuildStrategy::GradientScaleStrategy::kCoeffNumDevice)
      .value("One", BuildStrategy::GradientScaleStrategy::kOne)
      .value("Customized", BuildStrategy::GradientScaleStrategy::kCustomized);

  build_strategy.def(py::init())
      .def_property(
          "reduce_strategy",
          [](const BuildStrategy &self) { return self.reduce_; },
          [](BuildStrategy &self, BuildStrategy::ReduceStrategy strategy) {
X
Xin Pan 已提交
943
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
Y
yuyang18 已提交
944
            self.reduce_ = strategy;
C
chengduo 已提交
945 946 947 948 949 950 951
          },
          R"DOC(The type is STR, there are two reduce strategies in ParallelExecutor,
                  'AllReduce' and 'Reduce'. If you want that all the parameters'
                  optimization are done on all devices independently, you should choose 'AllReduce';
                  if you choose 'Reduce', all the parameters' optimization will be evenly distributed
                  to different devices, and then broadcast the optimized parameter to other devices.
                  In some models, `Reduce` is faster. Default 'AllReduce'. )DOC")
Y
yuyang18 已提交
952 953 954 955 956
      .def_property(
          "gradient_scale_strategy",
          [](const BuildStrategy &self) { return self.gradient_scale_; },
          [](BuildStrategy &self,
             BuildStrategy::GradientScaleStrategy strategy) {
X
Xin Pan 已提交
957
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
Y
yuyang18 已提交
958
            self.gradient_scale_ = strategy;
C
chengduo 已提交
959 960 961 962 963 964
          },
          R"DOC(The type is STR, there are three ways of defining :math:`loss@grad` in
                   ParallelExecutor, 'CoeffNumDevice', 'One' and 'Customized'. By default,
                   ParallelExecutor sets the :math:`loss@grad` according to the number of devices.
                   If you want to customize :math:`loss@grad`, you can choose 'Customized'.
                   Default 'CoeffNumDevice'.)DOC")
Y
yuyang18 已提交
965 966 967 968
      .def_property(
          "debug_graphviz_path",
          [](const BuildStrategy &self) { return self.debug_graphviz_path_; },
          [](BuildStrategy &self, const std::string &path) {
X
Xin Pan 已提交
969
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
Y
yuyang18 已提交
970
            self.debug_graphviz_path_ = path;
C
chengduo 已提交
971 972 973 974
          },
          R"DOC(The type is STR, debug_graphviz_path indicate the path that
                    writing the SSA Graph to file in the form of graphviz, you.
                    It is useful for debugging. Default "")DOC")
S
sneaxiy 已提交
975 976 977 978 979 980
      .def_property(
          "enable_sequential_execution",
          [](const BuildStrategy &self) {
            return self.enable_sequential_execution_;
          },
          [](BuildStrategy &self, bool b) {
X
Xin Pan 已提交
981
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
S
sneaxiy 已提交
982 983 984 985 986 987 988 989 990
            self.enable_sequential_execution_ = b;
          },
          R"DOC(The type is BOOL. If set True, the execution order of ops would be the same as what is in the program. Default False.)DOC")
      .def_property(
          "remove_unnecessary_lock",
          [](const BuildStrategy &self) {
            return self.remove_unnecessary_lock_;
          },
          [](BuildStrategy &self, bool b) {
X
Xin Pan 已提交
991
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
S
sneaxiy 已提交
992 993 994
            self.remove_unnecessary_lock_ = b;
          },
          R"DOC(The type is BOOL. If set True, some locks in GPU ops would be released and ParallelExecutor would run faster. Default False.)DOC")
995 996 997 998 999 1000
      .def_property(
          "num_trainers",
          [](const BuildStrategy &self) { return self.num_trainers_; },
          [](BuildStrategy &self, int num_trainers) {
            self.num_trainers_ = num_trainers;
          })
1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012
      .def_property(
          "trainers_endpoints",
          [](const BuildStrategy &self) { return self.trainers_endpoints_; },
          [](BuildStrategy &self,
             const std::vector<std::string> &trainers_endpoints) {
            self.trainers_endpoints_ = trainers_endpoints;
          })
      .def_property("trainer_id",
                    [](const BuildStrategy &self) { return self.trainer_id_; },
                    [](BuildStrategy &self, int trainer_id) {
                      self.trainer_id_ = trainer_id;
                    })
C
chengduo 已提交
1013 1014 1015 1016 1017 1018
      .def_property(
          "fuse_elewise_add_act_ops",
          [](const BuildStrategy &self) {
            return self.fuse_elewise_add_act_ops_;
          },
          [](BuildStrategy &self, bool b) {
X
Xin Pan 已提交
1019
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
C
chengduo 已提交
1020 1021 1022 1023 1024
            self.fuse_elewise_add_act_ops_ = b;
          },
          R"DOC(The type is BOOL, fuse_elewise_add_act_ops indicate whether
                     to fuse elementwise_add_op and activation_op,
                     it may make the execution faster. Default False)DOC")
D
dzhwinter 已提交
1025 1026 1027 1028
      .def_property(
          "memory_optimize",
          [](const BuildStrategy &self) { return self.memory_optimize_; },
          [](BuildStrategy &self, bool b) { self.memory_optimize_ = b; })
1029 1030 1031 1032
      .def_property(
          "is_distribution",
          [](const BuildStrategy &self) { return self.is_distribution_; },
          [](BuildStrategy &self, bool b) { self.is_distribution_ = b; })
Q
can run  
Qiao Longfei 已提交
1033 1034 1035
      .def_property("async_mode",
                    [](const BuildStrategy &self) { return self.async_mode_; },
                    [](BuildStrategy &self, bool b) { self.async_mode_ = b; })
D
dzhwinter 已提交
1036 1037 1038 1039
      .def_property(
          "memory_early_delete",
          [](const BuildStrategy &self) { return self.memory_early_delete_; },
          [](BuildStrategy &self, bool b) { self.memory_early_delete_ = b; })
1040
      .def("_finalize_strategy_and_create_passes",
X
fix  
Xin Pan 已提交
1041
           [](BuildStrategy &self) -> std::shared_ptr<ir::PassBuilder> {
1042 1043 1044 1045 1046
             return self.CreatePassesFromStrategy(true);
           },
           R"DOC(Allow user to customized passes. Normally model-specific
                optimization passes should be defined in this way. BuildStrategy
                cannot be updated after being finalized.)DOC");
Y
yuyang18 已提交
1047 1048 1049

  pe.def(py::init<const std::vector<platform::Place> &,
                  const std::unordered_set<std::string> &, const ProgramDesc &,
Y
yuyang18 已提交
1050
                  const std::string &, Scope *, std::vector<Scope *> &,
1051
                  const ExecutionStrategy &, const BuildStrategy &>())
Y
Yu Yang 已提交
1052 1053 1054 1055
      // NOTE: even we return a vec<Scope*>* to Python use reference policy.
      // We still cannot get local_scope from this vector, since the element
      // of vec<Scope*> will be freed by Python GC. We can only return Scope*
      // one by one and mark them as reference.
1056 1057 1058 1059 1060
      .def("local_scopes",
           [](ParallelExecutor &self) -> std::vector<Scope *> * {
             return &self.GetLocalScopes();
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
1061 1062 1063 1064
      .def("feed_tensors_into_local_scopes",
           &ParallelExecutor::FeedTensorsIntoLocalScopes)
      .def("feed_and_split_tensor_into_local_scopes",
           &ParallelExecutor::FeedAndSplitTensorIntoLocalScopes)
S
sneaxiy 已提交
1065 1066 1067 1068 1069 1070
      .def("run", [](ParallelExecutor &self,
                     const std::vector<std::string> &fetch_tensors,
                     const std::string &fetched_var_name) {
        pybind11::gil_scoped_release release;
        self.Run(fetch_tensors, fetched_var_name);
      });
Y
Yu Yang 已提交
1071

1072
  BindRecordIOWriter(&m);
W
Wang Guibao 已提交
1073
  BindAsyncExecutor(&m);
F
flame 已提交
1074 1075 1076

  BindGraph(&m);
  BindNode(&m);
L
Luo Tao 已提交
1077
}
1078
}  // namespace pybind
1079
}  // namespace paddle