pybind.cc 40.4 KB
Newer Older
1
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
2 3 4 5 6

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

7
http://www.apache.org/licenses/LICENSE-2.0
8 9 10 11 12 13

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
L
lgone2000 已提交
14
#include <Python.h>
C
chengduoZH 已提交
15 16
#include <algorithm>
#include <map>
S
sneaxiy 已提交
17
#include <memory>
C
chengduoZH 已提交
18 19 20 21 22
#include <mutex>  // NOLINT // for call_once
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>
23

Y
Yi Wang 已提交
24 25 26
#include "paddle/fluid/framework/executor.h"
#include "paddle/fluid/framework/feed_fetch_method.h"
#include "paddle/fluid/framework/framework.pb.h"
27
#include "paddle/fluid/framework/ir/pass_builder.h"
Y
Yi Wang 已提交
28 29 30
#include "paddle/fluid/framework/lod_rank_table.h"
#include "paddle/fluid/framework/lod_tensor.h"
#include "paddle/fluid/framework/lod_tensor_array.h"
31
#include "paddle/fluid/framework/op_registry.h"
Y
Yu Yang 已提交
32
#include "paddle/fluid/framework/parallel_executor.h"
Y
Yi Wang 已提交
33
#include "paddle/fluid/framework/prune.h"
Y
Refine  
Yu Yang 已提交
34
#include "paddle/fluid/framework/reader.h"
Y
Yi Wang 已提交
35
#include "paddle/fluid/framework/selected_rows.h"
X
Xin Pan 已提交
36
#include "paddle/fluid/framework/version.h"
Y
Refine  
Yu Yang 已提交
37
#include "paddle/fluid/memory/allocation/allocator_strategy.h"
D
dzhwinter 已提交
38
#include "paddle/fluid/operators/activation_op.h"
S
sneaxiy 已提交
39
#include "paddle/fluid/operators/reader/lod_tensor_blocking_queue.h"
Y
Yu Yang 已提交
40
#include "paddle/fluid/platform/cpu_info.h"
Y
Yi Wang 已提交
41
#include "paddle/fluid/platform/enforce.h"
42
#include "paddle/fluid/platform/init.h"
Y
Yi Wang 已提交
43 44
#include "paddle/fluid/platform/place.h"
#include "paddle/fluid/platform/profiler.h"
W
Wang Guibao 已提交
45
#include "paddle/fluid/pybind/async_executor_py.h"
Y
Yi Wang 已提交
46 47
#include "paddle/fluid/pybind/const_value.h"
#include "paddle/fluid/pybind/exception.h"
48 49
#include "paddle/fluid/pybind/protobuf.h"
#include "paddle/fluid/pybind/pybind.h"  // NOLINT
Y
Yu Yang 已提交
50
#include "paddle/fluid/pybind/recordio.h"
Y
Yi Wang 已提交
51
#include "paddle/fluid/pybind/tensor_py.h"
Y
Yu Yang 已提交
52

53
#include "paddle/fluid/string/to_string.h"
54

D
Dong Zhihong 已提交
55
#ifdef PADDLE_WITH_CUDA
P
peizhilin 已提交
56
#ifndef _WIN32
Y
Yi Wang 已提交
57
#include "paddle/fluid/operators/nccl/nccl_gpu_common.h"
P
peizhilin 已提交
58
#endif
Y
Yi Wang 已提交
59 60
#include "paddle/fluid/platform/cuda_profiler.h"
#include "paddle/fluid/platform/gpu_info.h"
D
Dong Zhihong 已提交
61 62
#endif

M
minqiyang 已提交
63 64
#include "pybind11/stl.h"

65 66 67 68
DEFINE_bool(reader_queue_speed_test_mode, false,
            "If set true, the queue.pop will only get data from queue but not "
            "remove the data from queue for speed testing");

Q
Qiao Longfei 已提交
69 70 71
// disable auto conversion to list in Python
PYBIND11_MAKE_OPAQUE(paddle::framework::LoDTensorArray);

72
namespace paddle {
73
namespace pybind {
74
bool IsCompiledWithCUDA() {
75
#ifndef PADDLE_WITH_CUDA
Q
qijun 已提交
76 77 78 79 80 81
  return false;
#else
  return true;
#endif
}

Y
update  
Yancey1989 已提交
82
bool IsCompiledWithDIST() {
Y
Yancey1989 已提交
83
#ifdef PADDLE_WITH_DISTRIBUTE
Y
update  
Yancey1989 已提交
84 85 86 87 88 89
  return true;
#else
  return false;
#endif
}

90
PYBIND11_MODULE(core, m) {
Y
Yu Yang 已提交
91 92 93
  // Not used, just make sure cpu_info.cc is linked.
  paddle::platform::CpuTotalPhysicalMemory();

Y
Refine  
Yu Yang 已提交
94
  paddle::memory::allocation::UseAllocatorStrategyGFlag();
95
  m.doc() = "C++ core of PaddlePaddle";
96

97 98 99 100
  // using framework in this function. Since it is inside a function, it will
  // not cause namespace pollution.
  using namespace paddle::framework;  // NOLINT

101
  BindException(&m);
Y
Yu Yang 已提交
102

103 104 105
  py::class_<Tensor>(m, "Tensor", py::buffer_protocol())
      .def_buffer(
          [](Tensor &self) -> py::buffer_info { return CastToPyBuffer(self); })
Y
yuyang18 已提交
106
      .def("_get_dims",
107
           [](const Tensor &self) { return vectorize(self.dims()); })
Y
yuyang18 已提交
108
      .def("_set_dims",
Q
qijun 已提交
109
           [](Tensor &self, const std::vector<int64_t> &dim) {
Y
Yu Yang 已提交
110
             self.Resize(make_ddim(dim));
Y
Yu Yang 已提交
111
           })
Y
yuyang18 已提交
112
      .def("_set_layout",
D
dzhwinter 已提交
113 114 115
           [](Tensor &self, const std::string &layout) {
             self.set_layout(StringToDataLayout(layout));
           })
Y
yuyang18 已提交
116
      .def("_alloc_float",
D
dzhwinter 已提交
117
           [](Tensor &self, paddle::platform::CUDAPlace &place) {
Q
qijun 已提交
118
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
119
           })
Y
yuyang18 已提交
120
      .def("_alloc_float",
Y
Yu Yang 已提交
121
           [](Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
122
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
123
           })
Y
yuyang18 已提交
124
      .def("_alloc_int",
Y
Yu Yang 已提交
125
           [](Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
126
             self.mutable_data<int>(place);
Y
Yu Yang 已提交
127
           })
Y
yuyang18 已提交
128
      .def("_alloc_int",
D
dzhwinter 已提交
129
           [](Tensor &self, paddle::platform::CUDAPlace &place) {
Q
qijun 已提交
130
             self.mutable_data<int>(place);
Q
qijun 已提交
131
           })
Y
yuyang18 已提交
132
      .def("_alloc_int",
C
chengduoZH 已提交
133 134 135
           [](Tensor &self, paddle::platform::CUDAPinnedPlace &place) {
             self.mutable_data<int>(place);
           })
Y
yuyang18 已提交
136
      .def("_alloc_float",
C
chengduoZH 已提交
137 138 139
           [](Tensor &self, paddle::platform::CUDAPinnedPlace &place) {
             self.mutable_data<float>(place);
           })
Y
Yu Yang 已提交
140 141
      .def("set", PyCPUTensorSetFromArray<float>)
      .def("set", PyCPUTensorSetFromArray<int>)
142
      .def("set", PyCPUTensorSetFromArray<double>)
143
      .def("set", PyCPUTensorSetFromArray<int64_t>)
Y
Yu Yang 已提交
144
      .def("set", PyCPUTensorSetFromArray<bool>)
145
      .def("set", PyCPUTensorSetFromArray<uint16_t>)
F
fengjiayi 已提交
146
      .def("set", PyCPUTensorSetFromArray<uint8_t>)
Q
qingqing01 已提交
147
      .def("set", PyCPUTensorSetFromArray<int8_t>)
148
#ifdef PADDLE_WITH_CUDA
Y
Yu Yang 已提交
149 150
      .def("set", PyCUDATensorSetFromArray<float>)
      .def("set", PyCUDATensorSetFromArray<int>)
151
      .def("set", PyCUDATensorSetFromArray<double>)
152
      .def("set", PyCUDATensorSetFromArray<int64_t>)
Y
Yu Yang 已提交
153
      .def("set", PyCUDATensorSetFromArray<bool>)
154
      .def("set", PyCUDATensorSetFromArray<uint16_t>)
F
fengjiayi 已提交
155
      .def("set", PyCUDATensorSetFromArray<uint8_t>)
Q
qingqing01 已提交
156
      .def("set", PyCUDATensorSetFromArray<int8_t>)
C
chengduoZH 已提交
157 158 159 160 161 162
      .def("set", PyCUDAPinnedTensorSetFromArray<float>)
      .def("set", PyCUDAPinnedTensorSetFromArray<int>)
      .def("set", PyCUDAPinnedTensorSetFromArray<double>)
      .def("set", PyCUDAPinnedTensorSetFromArray<int64_t>)
      .def("set", PyCUDAPinnedTensorSetFromArray<bool>)
      .def("set", PyCUDAPinnedTensorSetFromArray<uint16_t>)
F
fengjiayi 已提交
163
      .def("set", PyCUDAPinnedTensorSetFromArray<uint8_t>)
Q
qingqing01 已提交
164
      .def("set", PyCUDAPinnedTensorSetFromArray<int8_t>)
Q
qijun 已提交
165
#endif
166
      .def("shape", [](Tensor &self) { return vectorize(self.dims()); })
Y
yuyang18 已提交
167 168 169 170 171
      .def("_set_float_element", TensorSetElement<float>)
      .def("_get_float_element", TensorGetElement<float>)
      .def("_set_double_element", TensorSetElement<double>)
      .def("_get_double_element", TensorGetElement<double>)
      .def("_dtype", [](Tensor &self) { return ToDataType(self.type()); });
Y
Yu Yang 已提交
172

X
Xin Pan 已提交
173 174 175 176 177 178 179 180 181 182 183 184 185
  py::class_<LoDTensor, Tensor>(m, "LoDTensor", R"DOC(
    LoDTensor is a Tensor with optional LoD information.

    np.array(lod_tensor) can convert LoDTensor to numpy array.
    lod_tensor.lod() can retrieve the LoD information.

    LoD is short for Level of Details and is usually used for varied sequence
    length. You can skip the following comment if you don't need optional LoD.

  For example:
     A LoDTensor X can look like the example below. It contains 2 sequences.
     The first has length 2 and the second has length 3, as described by x.lod.

X
fix doc  
Xin Pan 已提交
186
     The first tensor dimension 5=2+3 is calculated from LoD if it's available.
X
Xin Pan 已提交
187
     It means the total number of sequence element. In X, each element has 2
X
fix doc  
Xin Pan 已提交
188
     columns, hence [5, 2].
X
Xin Pan 已提交
189 190 191

      x.lod  = [[2, 3]]
      x.data = [[1, 2], [3, 4],
X
fix doc  
Xin Pan 已提交
192 193
                [5, 6], [7, 8], [9, 10]]
      x.shape = [5, 2]
X
Xin Pan 已提交
194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216

      LoD can have multiple levels (for example, a paragraph can have multiple
      sentences and a sentence can have multiple words). In the following
      LodTensor Y, the lod_level is 2. It means there are 2 sequence, the
      first sequence length is 2 (has 2 sub-sequences), the second one's
      length is 1. The first sequence's 2 sub-sequences have length 2 and 2,
      respectively. And the second sequence's 1 sub-sequence has length 3.

      y.lod = [[2 1], [2 2 3]]
      y.shape = [2+2+3, ...]

  Note:
      In above description, LoD is length-based. In Paddle internal
      implementation, lod is offset-based. Hence, internally,
      y.lod is represented as [[0, 2, 3], [0, 2, 4, 7]] (length-based
      equivlent would be [[2-0, 3-2], [2-0, 4-2, 7-4]]).

      Sometimes LoD is called recursive_sequence_length to be more
      self-explanatory. In this case, it must be length-based. Due to history
      reasons. when LoD is called lod in public API, it might be offset-based.
      Users should be careful about it.

        )DOC")
217 218
      .def_buffer(
          [](Tensor &self) -> py::buffer_info { return CastToPyBuffer(self); })
219 220 221 222 223 224 225 226 227 228 229 230 231 232
      .def("__init__",
           [](LoDTensor &instance, const std::vector<std::vector<size_t>>
                                       &recursive_sequence_lengths) {
             LoD new_lod;
             new_lod.reserve(recursive_sequence_lengths.size());
             std::copy(recursive_sequence_lengths.begin(),
                       recursive_sequence_lengths.end(),
                       std::back_inserter(new_lod));
             LoD new_offset_lod = ConvertToOffsetBasedLoD(new_lod);
             PADDLE_ENFORCE(
                 CheckLoD(new_offset_lod, -1),
                 "the provided recursive_sequence_lengths info is invalid");
             new (&instance) LoDTensor(new_offset_lod);
           })
Y
Yu Yang 已提交
233
      .def("__init__", [](LoDTensor &instance) { new (&instance) LoDTensor(); })
G
gongweibao 已提交
234 235 236 237 238
      // We implement offset based LOD in C++ while we use length based with
      // Python API. So we changed set_lod to set_recursive_sequence_lengths to
      // avoid misuse.
      // The discussion is here:
      // https://github.com/PaddlePaddle/Paddle/issues/10855
D
dangqingqing 已提交
239
      .def("set_lod",
240
           [](LoDTensor &self, const std::vector<std::vector<size_t>> &lod) {
241
             // the input lod is offset-based level-of-detail info
Y
Yu Yang 已提交
242
             LoD new_lod;
243 244
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
245 246
             PADDLE_ENFORCE(CheckLoD(new_lod, vectorize(self.dims()).front()),
                            "the provided lod info is invalid");
247
             self.set_lod(new_lod);
D
dangqingqing 已提交
248
           })
249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273
      .def("set_recursive_sequence_lengths",
           [](LoDTensor &self, const std::vector<std::vector<size_t>>
                                   &recursive_sequence_lengths) {
             // the input recursive_sequence_lengths is length-based
             // level-of-detail info
             LoD new_lod;
             new_lod.reserve(recursive_sequence_lengths.size());
             std::copy(recursive_sequence_lengths.begin(),
                       recursive_sequence_lengths.end(),
                       std::back_inserter(new_lod));
             LoD new_offset_lod = ConvertToOffsetBasedLoD(new_lod);
             PADDLE_ENFORCE(
                 CheckLoD(new_offset_lod, vectorize(self.dims()).front()),
                 "the provided recursive_sequence_lengths info is invalid");
             self.set_lod(new_offset_lod);
           })
      .def("lod",
           [](LoDTensor &self) -> std::vector<std::vector<size_t>> {
             // output the offset-based lod info
             LoD lod = self.lod();
             std::vector<std::vector<size_t>> new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             return new_lod;
           })
G
gongweibao 已提交
274
      // Set above comments of set_lod.
275 276 277 278 279 280 281 282 283 284 285 286 287
      .def("recursive_sequence_lengths",
           [](LoDTensor &self) -> std::vector<std::vector<size_t>> {
             // output the length-based lod info
             LoD lod = ConvertToLengthBasedLoD(self.lod());
             std::vector<std::vector<size_t>> new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             return new_lod;
           })
      .def("has_valid_recursive_sequence_lengths", [](LoDTensor &self) -> bool {
        // Check that the lod info is valid and match the outermost
        // dimension of the LoDTensor data
        return CheckLoD(self.lod(), vectorize(self.dims()).front());
D
dangqingqing 已提交
288 289
      });

Q
qijun 已提交
290 291 292 293 294 295 296 297 298 299 300 301 302
  py::class_<SelectedRows>(m, "SelectedRows")
      .def("__init__",
           [](SelectedRows &instance) { new (&instance) SelectedRows(); })
      .def("__init__",
           [](SelectedRows &instance, const std::vector<int64_t> rows,
              const int64_t &height) {
             new (&instance) SelectedRows(rows, height);
           })
      .def("get_tensor",
           [](SelectedRows &self) { return self.mutable_value(); },
           py::return_value_policy::reference)
      .def("set_height", &SelectedRows::set_height)
      .def("height", &SelectedRows::height)
Q
qijun 已提交
303 304 305 306 307 308 309 310 311
      .def("set_rows",
           [](SelectedRows &self, std::vector<int64_t> rows) {
#ifndef PADDLE_WITH_CUDA
             self.set_rows(rows);
#else
        Vector<int64_t> new_rows(rows);
        self.set_rows(new_rows);
#endif
           })
312
      .def("sync_index", [](SelectedRows &instance) { instance.SyncIndex(); })
313
      .def("rows", [](SelectedRows &self) {
314 315 316 317 318
        auto rows = self.rows();
        std::vector<int64_t> new_rows;
        new_rows.reserve(rows.size());
        std::copy(rows.begin(), rows.end(), std::back_inserter(new_rows));
        return new_rows;
319
      });
Q
qijun 已提交
320

321
  py::class_<Variable>(m, "Variable", R"DOC(Variable Class.
322 323 324

All parameter, weight, gradient are variables in Paddle.
)DOC")
325
      .def("is_int", [](const Variable &var) { return var.IsType<int>(); })
326
      .def("set_int",
327 328
           [](Variable &var, int val) -> void { *var.GetMutable<int>() = val; })
      .def("get_int", [](const Variable &var) -> int { return var.Get<int>(); })
329 330 331 332 333 334 335
      .def("is_float", [](const Variable &var) { return var.IsType<float>(); })
      .def("set_float",
           [](Variable &var, float val) -> void {
             *var.GetMutable<float>() = val;
           })
      .def("get_float",
           [](const Variable &var) -> float { return var.Get<float>(); })
Y
Yu Yang 已提交
336
      .def("get_tensor",
337 338
           [](Variable &self) -> LoDTensor * {
             return self.GetMutable<LoDTensor>();
D
dangqingqing 已提交
339 340
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
341 342 343
      .def("get_lod_rank_table",
           [](Variable &self) { return self.GetMutable<LoDRankTable>(); },
           py::return_value_policy::reference)
Q
qijun 已提交
344 345 346 347 348
      .def("get_selected_rows",
           [](Variable &self) -> SelectedRows * {
             return self.GetMutable<SelectedRows>();
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
349 350 351
      .def("get_lod_tensor_array",
           [](Variable &self) { return self.GetMutable<LoDTensorArray>(); },
           py::return_value_policy::reference)
P
peizhilin 已提交
352
#if (defined(PADDLE_WITH_CUDA) && !defined(_WIN32))
D
Dong Zhihong 已提交
353 354 355 356 357
      .def("get_communicator",
           [](Variable &self) -> platform::Communicator * {
             return self.GetMutable<platform::Communicator>();
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
358
#endif
Y
Refine  
Yu Yang 已提交
359 360 361 362 363
      .def("get_reader",
           [](Variable &self) -> framework::ReaderHolder * {
             PADDLE_ENFORCE(self.IsType<framework::ReaderHolder>());
             return self.GetMutable<framework::ReaderHolder>();
           },
W
wopeizl 已提交
364
           py::return_value_policy::reference);
365

Y
Refine  
Yu Yang 已提交
366
  py::class_<framework::ReaderHolder>(m, "Reader", "")
367
      .def("reset", &framework::ReaderHolder::ResetAll);
Y
Refine  
Yu Yang 已提交
368

S
sneaxiy 已提交
369 370 371 372
  using LoDTensorBlockingQueue =
      ::paddle::operators::reader::LoDTensorBlockingQueue;
  using LoDTensorBlockingQueueHolder =
      ::paddle::operators::reader::LoDTensorBlockingQueueHolder;
S
sneaxiy 已提交
373 374
  py::class_<LoDTensorBlockingQueue, std::shared_ptr<LoDTensorBlockingQueue>>(
      m, "LoDTensorBlockingQueue", "")
S
sneaxiy 已提交
375
      .def("push",
S
sneaxiy 已提交
376
           [](LoDTensorBlockingQueue &self,
S
sneaxiy 已提交
377
              const std::vector<framework::LoDTensor> &lod_tensor_vec) {
S
sneaxiy 已提交
378
             pybind11::gil_scoped_release release;
S
sneaxiy 已提交
379
             return self.Push(lod_tensor_vec);
S
sneaxiy 已提交
380
           })
S
sneaxiy 已提交
381 382 383 384
      .def("size", &LoDTensorBlockingQueue::Size)
      .def("capacity", &LoDTensorBlockingQueue::Cap)
      .def("close", &LoDTensorBlockingQueue::Close)
      .def("is_closed", &LoDTensorBlockingQueue::IsClosed);
S
sneaxiy 已提交
385

S
sneaxiy 已提交
386
  m.def("init_lod_tensor_blocking_queue",
S
sneaxiy 已提交
387
        [](Variable &var, size_t capacity,
S
sneaxiy 已提交
388
           const std::vector<std::vector<int64_t>> &shapes)
S
sneaxiy 已提交
389
            -> std::shared_ptr<LoDTensorBlockingQueue> {
S
sneaxiy 已提交
390 391 392 393 394 395
              std::vector<DDim> dims(shapes.size());
              std::transform(shapes.begin(), shapes.end(), dims.begin(),
                             [](const std::vector<int64_t> &shape) {
                               return make_ddim(shape);
                             });
              auto *holder = var.GetMutable<LoDTensorBlockingQueueHolder>();
396 397
              holder->InitOnce(capacity, dims,
                               FLAGS_reader_queue_speed_test_mode);
S
sneaxiy 已提交
398
              return holder->GetQueue();
S
sneaxiy 已提交
399
            },
S
sneaxiy 已提交
400
        py::return_value_policy::copy);
S
sneaxiy 已提交
401

Q
Qiao Longfei 已提交
402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421
  py::class_<Scope>(m, "Scope", R"DOC(
    Scope is an association of a name to Variable. All variables belong to Scope.

    Variables in a parent scope can be retrieved from local scope.

    You need to specify a scope to run a Net, i.e., `exe.Run(&scope)`.
    One net can run in different scopes and update different variable in the
    scope.

    You can create var in a scope and get it from the scope.

    Examples:
        .. code-block:: python

          # create tensor from a scope and set value to it.
          param = scope.var('Param').get_tensor()
          param_array = np.full((height, row_numel), 5.0).astype("float32")
          param.set(param_array, place)

        )DOC")
D
dongzhihong 已提交
422
      .def("var",
423
           [](Scope &self, const std::string &name) -> Variable * {
D
dongzhihong 已提交
424
             return self.Var(name);
Y
Yu Yang 已提交
425
           },
426
           py::return_value_policy::reference)
427
      .def("find_var", &Scope::FindVar, py::return_value_policy::reference)
Y
Yu Yang 已提交
428
      .def(py::init<>())
429
      .def("new_scope", [](Scope &self) -> Scope * { return &self.NewScope(); },
430
           py::return_value_policy::reference)
Y
Yu Yang 已提交
431
      .def("drop_kids", &Scope::DropKids);
432

Y
Yu Yang 已提交
433 434
  //! @note: Be careful! PyBind will return std::string as an unicode, not
  //! Python str. If you want a str object, you should cast them in Python.
Y
Yu Yang 已提交
435 436
  m.def("get_all_op_protos", []() -> std::vector<py::bytes> {
    std::vector<py::bytes> ret_values;
437 438 439 440 441 442 443 444 445 446
    for (auto &iter : OpInfoMap::Instance().map()) {
      auto &info = iter.second;
      if (info.HasOpProtoAndChecker()) {
        std::string str;
        PADDLE_ENFORCE(
            info.Proto().SerializeToString(&str),
            "Serialize OpProto Error. This could be a bug of Paddle.");
        ret_values.emplace_back(str);
      }
    }
Y
Yu Yang 已提交
447 448
    return ret_values;
  });
449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464
  m.def(
      "get_grad_op_desc", [](const OpDesc &op_desc,
                             const std::unordered_set<std::string> &no_grad_set,
                             const std::vector<BlockDesc *> &grad_sub_block) {
        std::unordered_map<std::string, std::string> grad_to_var;
        std::vector<std::unique_ptr<OpDesc>> grad_op_descs =
            framework::OpInfoMap::Instance()
                .Get(op_desc.Type())
                .GradOpMaker()(op_desc, no_grad_set, &grad_to_var,
                               grad_sub_block);
        std::vector<OpDesc *> grad_op_desc_ptrs(grad_op_descs.size());
        std::transform(grad_op_descs.begin(), grad_op_descs.end(),
                       grad_op_desc_ptrs.begin(),
                       [](std::unique_ptr<OpDesc> &p) { return p.release(); });
        return std::make_pair(grad_op_desc_ptrs, grad_to_var);
      });
Y
Yu Yang 已提交
465
  m.def("prune", [](const ProgramDesc &origin,
466
                    const std::vector<std::array<size_t, 2>> &targets) {
Y
Yu Yang 已提交
467
    ProgramDesc prog_with_targets(origin);
468
    for (const auto &t : targets) {
469
      prog_with_targets.MutableBlock(t[0])->Op(t[1])->SetIsTarget(true);
470
    }
471
    proto::ProgramDesc pruned_desc;
472
    Prune(*prog_with_targets.Proto(), &pruned_desc);
Y
Yu Yang 已提交
473
    return new ProgramDesc(pruned_desc);
474
  });
475 476 477 478
  m.def("empty_var_name",
        []() { return std::string(framework::kEmptyVarName); });
  m.def("grad_var_suffix",
        []() { return std::string(framework::kGradVarSuffix); });
479 480 481
  m.def_submodule(
       "var_names",
       "The module will return special predefined variable name in Paddle")
Y
Yi Wang 已提交
482 483
      .def("empty", []() { return kEmptyVarName; })
      .def("temp", []() { return kTempVarName; });
Q
qijun 已提交
484
  // clang-format off
Y
Yu Yang 已提交
485
  py::class_<paddle::platform::DeviceContext>(m, "DeviceContext")
Q
qijun 已提交
486 487
      .def_static("create",
                  [](paddle::platform::CPUPlace& place)
Q
qijun 已提交
488
                      -> paddle::platform::DeviceContext* {
Q
qijun 已提交
489 490 491
                    return new paddle::platform::CPUDeviceContext();
                  })
      .def_static("create",
D
dzhwinter 已提交
492
                  [](paddle::platform::CUDAPlace& place)
Q
qijun 已提交
493
                      -> paddle::platform::DeviceContext* {
494
#ifndef PADDLE_WITH_CUDA
D
dzhwinter 已提交
495
                    PADDLE_THROW("CUDAPlace is not supported in CPU device.");
Q
qijun 已提交
496
#else
Q
qijun 已提交
497
                    return new paddle::platform::CUDADeviceContext(place);
Q
qijun 已提交
498
#endif
C
chengduoZH 已提交
499 500 501 502 503 504 505 506 507 508 509
                  })
          .def_static("create",
                [](paddle::platform::CUDAPinnedPlace& place)
                        -> paddle::platform::DeviceContext* {
#ifndef PADDLE_WITH_CUDA
                  PADDLE_THROW(
                        "CUDAPinnedPlace is not supported in CPU device.");
#else
                  return new paddle::platform::CUDAPinnedDeviceContext(place);
#endif
                });;
D
Dong Zhihong 已提交
510
// clang-format on
P
peizhilin 已提交
511
#if (defined(PADDLE_WITH_CUDA) && !defined(_WIN32))
D
Dong Zhihong 已提交
512 513
  py::class_<platform::Communicator>(m, "Communicator").def(py::init<>());
#endif
D
dzhwinter 已提交
514
  py::class_<platform::CUDAPlace>(m, "CUDAPlace")
515
      .def(py::init<int>())
D
dzhwinter 已提交
516
      .def("__str__", string::to_string<const platform::CUDAPlace &>);
Q
qijun 已提交
517

518 519 520
  py::class_<paddle::platform::CPUPlace>(m, "CPUPlace")
      .def(py::init<>())
      .def("__str__", string::to_string<const platform::CPUPlace &>);
Y
Yu Yang 已提交
521

C
chengduoZH 已提交
522 523 524 525
  py::class_<paddle::platform::CUDAPinnedPlace>(m, "CUDAPinnedPlace")
      .def(py::init<>())
      .def("__str__", string::to_string<const platform::CUDAPinnedPlace &>);

Y
Yu Yang 已提交
526 527 528 529 530 531 532
  py::class_<platform::Place>(m, "Place")
      .def(py::init<>())
      .def("set_place",
           [](platform::Place &self, const platform::CPUPlace &cpu_place) {
             self = cpu_place;
           })
      .def("set_place",
D
dzhwinter 已提交
533
           [](platform::Place &self, const platform::CUDAPlace &gpu_place) {
Y
Yu Yang 已提交
534
             self = gpu_place;
C
chengduoZH 已提交
535 536
           })
      .def("set_place", [](platform::Place &self,
C
chengduoZH 已提交
537 538
                           const platform::CUDAPinnedPlace &cuda_pinned_place) {
        self = cuda_pinned_place;
C
chengduoZH 已提交
539
      });
Y
Yu Yang 已提交
540

Y
Yu Yang 已提交
541 542 543
  py::class_<OperatorBase>(m, "Operator")
      .def_static("create",
                  [](py::bytes protobin) {
544
                    proto::OpDesc desc;
Y
Yu Yang 已提交
545 546 547 548 549
                    PADDLE_ENFORCE(desc.ParsePartialFromString(protobin),
                                   "Cannot parse user input to OpDesc");
                    PADDLE_ENFORCE(desc.IsInitialized(),
                                   "User OpDesc is not initialized, reason %s",
                                   desc.InitializationErrorString());
550
                    return OpRegistry::CreateOp(desc);
Y
Yu Yang 已提交
551
                  })
552
      .def("run",
553
           [](OperatorBase &self, const Scope &scope,
D
dzhwinter 已提交
554 555 556
              const platform::CPUPlace &place) { self.Run(scope, place); })
      .def("run",
           [](OperatorBase &self, const Scope &scope,
D
dzhwinter 已提交
557
              const platform::CUDAPlace &place) { self.Run(scope, place); })
C
chengduoZH 已提交
558 559 560 561 562
      .def("run",
           [](OperatorBase &self, const Scope &scope,
              const platform::CUDAPinnedPlace &place) {
             self.Run(scope, place);
           })
Y
Yu Yang 已提交
563 564 565 566 567 568 569
      .def("type",
           [](const OperatorBase &op) -> std::string { return op.Type(); })
      .def("outputs",
           [](const OperatorBase &op)
               -> std::map<std::string, std::vector<std::string>> {
                 return op.Outputs();
               })
Q
qijun 已提交
570 571
      .def("output_vars",
           [](const OperatorBase &op) { return op.OutputVars(true); })
Y
Yu Yang 已提交
572
      .def("inputs", [](const OperatorBase &op) { return op.Inputs(); })
Q
qijun 已提交
573
      .def("input_vars", [](const OperatorBase &op) { return op.InputVars(); })
Y
Yu Yang 已提交
574 575 576 577
      .def("__str__", &OperatorBase::DebugString)
      .def("no_intermediate_outputs",
           [](const OperatorBase &op) { return op.OutputVars(false); })
      .def("support_gpu", &OperatorBase::SupportGPU);
Y
Yu Yang 已提交
578

F
fengjiayi 已提交
579
  py::class_<framework::Executor>(m, "Executor")
D
dzhwinter 已提交
580
      .def(py::init<const platform::Place &>())
Y
Yancey1989 已提交
581
      .def("close", &Executor::Close)
S
sneaxiy 已提交
582 583 584 585 586
      .def("run", [](Executor &self, const ProgramDesc &prog, Scope *scope,
                     int block_id, bool create_local_scope, bool create_vars) {
        pybind11::gil_scoped_release release;
        self.Run(prog, scope, block_id, create_local_scope, create_vars);
      });
S
sneaxiy 已提交
587

D
dzhwinter 已提交
588
  m.def("init_gflags", framework::InitGflags);
Y
Yang Yu 已提交
589
  m.def("init_glog", framework::InitGLOG);
X
Xin Pan 已提交
590 591
  m.def("init_devices",
        [](bool init_p2p) { framework::InitDevices(init_p2p); });
592

593
  m.def("is_compiled_with_cuda", IsCompiledWithCUDA);
Y
update  
Yancey1989 已提交
594
  m.def("is_compiled_with_dist", IsCompiledWithDIST);
595 596 597 598 599 600
#ifdef PADDLE_WITH_CUDA
  m.def("is_float16_supported", [](const platform::CUDAPlace &place) -> bool {
    // Only GPUs with Compute Capability >= 53 support float16
    return platform::GetCUDAComputeCapability(place.device) >= 53;
  });
#endif
601

602
  m.def("set_feed_variable", framework::SetFeedVariable);
Q
qijun 已提交
603
  m.def("get_fetch_variable", framework::GetFetchVariable);
Q
qijun 已提交
604

X
Xin Pan 已提交
605 606
  m.def("_is_program_version_supported", IsProgramVersionSupported);

607 608 609 610 611
  BindProgramDesc(&m);
  BindBlockDesc(&m);
  BindVarDsec(&m);
  BindOpDesc(&m);
  BindConstValue(&m);
Y
Yu Yang 已提交
612

Y
Yu Yang 已提交
613 614 615 616 617 618 619 620 621
  py::class_<framework::LoDRankTable>(m, "LodRankTable")
      .def("items", [](framework::LoDRankTable &table) {
        std::vector<std::pair<size_t, size_t>> res;
        for (auto &item : table.items()) {
          res.push_back({item.index, item.length});
        }
        return res;
      });

Y
Yu Yang 已提交
622
  py::class_<LoDTensorArray>(m, "LoDTensorArray")
S
sneaxiy 已提交
623 624
      .def("__init__",
           [](LoDTensorArray &instance) { new (&instance) LoDTensorArray(); })
Y
Yu Yang 已提交
625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640
      .def("__getitem__",
           [](LoDTensorArray &self, size_t i) { return &self.at(i); },
           py::return_value_policy::reference)
      .def("__len__", [](LoDTensorArray &self) { return self.size(); })
      .def("__setitem__",
           [](LoDTensorArray &self, size_t i, const LoDTensor &t) {
             PADDLE_ENFORCE_LT(i, self.size());
             self[i].ShareDataWith(t);
             self[i].set_lod(t.lod());
           })
      .def("append", [](LoDTensorArray &self, const LoDTensor &t) {
        self.emplace_back();
        self.back().ShareDataWith(t);
        self.back().set_lod(t.lod());
      });

D
dzhwinter 已提交
641 642 643
  m.def("IsInplace",
        [](std::string op) -> bool { return operators::IsInplace(op); });

Y
Yu Yang 已提交
644
  m.def("op_support_gpu", OpSupportGPU);
D
Dong Zhihong 已提交
645
#ifdef PADDLE_WITH_CUDA
D
Dong Zhihong 已提交
646
  m.def("get_cuda_device_count", platform::GetCUDADeviceCount);
D
dangqingqing 已提交
647

P
peizhilin 已提交
648
#ifndef _WIN32
D
dangqingqing 已提交
649 650 651
  m.def("nvprof_init", platform::CudaProfilerInit);
  m.def("nvprof_start", platform::CudaProfilerStart);
  m.def("nvprof_stop", platform::CudaProfilerStop);
D
Dong Zhihong 已提交
652
#endif
P
peizhilin 已提交
653
#endif
Y
Yu Yang 已提交
654

655 656 657 658
  py::enum_<platform::ProfilerState>(m, "ProfilerState", py::arithmetic())
      .value("kDisabled", platform::ProfilerState::kDisabled)
      .value("kCPU", platform::ProfilerState::kCPU)
      .value("kCUDA", platform::ProfilerState::kCUDA)
659
      .value("kAll", platform::ProfilerState::kAll)
660 661 662 663 664 665 666 667 668 669 670 671 672
      .export_values();

  py::enum_<platform::EventSortingKey>(m, "EventSortingKey", py::arithmetic())
      .value("kDefault", platform::EventSortingKey::kDefault)
      .value("kCalls", platform::EventSortingKey::kCalls)
      .value("kTotal", platform::EventSortingKey::kTotal)
      .value("kMin", platform::EventSortingKey::kMin)
      .value("kMax", platform::EventSortingKey::kMax)
      .value("kAve", platform::EventSortingKey::kAve)
      .export_values();

  m.def("enable_profiler", platform::EnableProfiler);
  m.def("disable_profiler", platform::DisableProfiler);
X
Xin Pan 已提交
673
  m.def("is_profiler_enabled", platform::IsProfileEnabled);
674
  m.def("reset_profiler", platform::ResetProfiler);
Y
Yu Yang 已提交
675

676 677
  py::class_<ir::Pass, std::shared_ptr<ir::Pass>> pass(m, "Pass");
  pass.def(py::init())
678 679 680 681 682
      .def(
          "set_str",
          [](ir::Pass &self, const std::string &name, const std::string &attr) {
            self.Set<std::string>(name, new std::string(attr));
          })
X
Xin Pan 已提交
683 684 685
      .def("set_int", [](ir::Pass &self, const std::string &name,
                         int val) { self.Set<const int>(name, new int(val)); })
      .def("type", &ir::Pass::Type);
686

X
fix  
Xin Pan 已提交
687 688
  py::class_<ir::PassBuilder, std::shared_ptr<ir::PassBuilder>> pb(
      m, "PassBuilder");
689 690 691 692 693 694 695 696 697 698 699 700 701 702
  pb.def(py::init())
      .def("append_pass",
           [](ir::PassBuilder &self,
              const std::string &pass_type) -> std::shared_ptr<ir::Pass> {
             return self.AppendPass(pass_type);
           })
      .def("all_passes", [](ir::PassBuilder &self) { return self.AllPasses(); })
      .def("insert_pass",
           [](ir::PassBuilder &self, size_t idx, const std::string &pass_type) {
             return self.InsertPass(idx, pass_type);
           })
      .def("remove_pass",
           [](ir::PassBuilder &self, size_t idx) { self.RemovePass(idx); });

Y
yuyang18 已提交
703
  // -- python binds for parallel executor.
Y
yuyang18 已提交
704
  py::class_<ParallelExecutor> pe(m, "ParallelExecutor");
C
chengduo 已提交
705 706 707 708
  py::class_<ExecutionStrategy> exec_strategy(pe, "ExecutionStrategy", R"DOC(
    ExecutionStrategy allows the user to more preciously control how to run
    the program in ParallelExecutor by setting the property.

C
chengduo 已提交
709 710 711 712 713 714 715 716 717 718 719
    Examples:
        .. code-block:: python

          exec_strategy = fluid.ExecutionStrategy()
          exec_strategy.num_threads = 4

          train_exe = fluid.ParallelExecutor(use_cuda=True,
                                             loss_name=loss.name,
                                             exec_strategy=exec_strategy)

          train_loss, = train_exe.run([loss.name], feed=feed_dict)
C
chengduo 已提交
720 721 722

        )DOC");

Y
yuyang18 已提交
723
  exec_strategy.def(py::init())
Y
yuyang18 已提交
724 725 726 727 728
      .def_property(
          "num_threads",
          [](const ExecutionStrategy &self) { return self.num_threads_; },
          [](ExecutionStrategy &self, size_t num_threads) {
            self.num_threads_ = num_threads;
C
chengduo 已提交
729 730 731 732 733 734 735 736 737 738
          },
          R"DOC(The type is INT, num_threads represents the size of thread pool that
            used to run the operators of the current program in ParallelExecutor.
            If :math:`num\_threads=1`, all the operators will execute one by one,
            but the order maybe difference between iterations.
            If it is not set, it will be set in ParallelExecutor according to the
            device type and device count, for GPU, :math:`num\_threads=device\_count*4`, for CPU,
            :math:`num\_threads=CPU\_NUM*4`, the explanation of:math:`CPU\_NUM` is in ParallelExecutor.
            if it is not set, ParallelExecutor will get the cpu count by calling
            `multiprocessing.cpu_count()`. Default 0.)DOC")
Y
yuyang18 已提交
739
      .def_property(
740 741 742 743
          "use_cuda",
          [](const ExecutionStrategy &self) { return self.use_cuda_; },
          [](ExecutionStrategy &self, bool use_cuda) {
            self.use_cuda_ = use_cuda;
C
chengduo 已提交
744 745 746 747
          })  // FIXME(chengduo): Doesn't add doc for 'use_cuda', use_cuda may
      // make user confuse, because ParallelExecutor has a parameter named
      // 'use_cuda' too, in current implementation, ParallelExecutor's
      // 'use_cuda' will rewrite ExecutionStrategy's 'use_cuda'.
Y
yuyang18 已提交
748 749 750 751 752
      .def_property(
          "allow_op_delay",
          [](const ExecutionStrategy &self) { return self.allow_op_delay_; },
          [](ExecutionStrategy &self, bool allow_op_delay) {
            self.allow_op_delay_ = allow_op_delay;
C
chengduo 已提交
753 754 755 756
          },
          R"DOC(The type is BOOL, allow_op_delay represents whether to delay the
                communication operators to run, it may make the execution faster.
                Note that in some models, allow_op_delay may cause program hang. Default False.)DOC")
Y
yuyang18 已提交
757 758 759 760 761 762 763
      .def_property(
          "num_iteration_per_drop_scope",
          [](const ExecutionStrategy &self) {
            return self.num_iteration_per_drop_scope_;
          },
          [](ExecutionStrategy &self, size_t num_iteration_per_drop_scope) {
            self.num_iteration_per_drop_scope_ = num_iteration_per_drop_scope;
C
chengduo 已提交
764 765 766 767 768 769 770 771 772 773 774
          },
          R"DOC(The type is INT, num_iteration_per_drop_scope indicates how
                many iterations to clean up the temp variables which
                is generated during execution. It may make the execution faster,
                because the temp variable's shape maybe the same between two iterations. Default 100.

                NOTES:
                    1. If you fetch data when calling the 'run', the ParallelExecutor
                       will clean up the temp variables at the end of the current iteration.
                    2. In some NLP model, it may cause the GPU memory is insufficient,
                       in this case, you should reduce `num_iteration_per_drop_scope`.
775 776 777 778 779 780
              )DOC")
      .def_property("_dry_run",
                    [](const ExecutionStrategy &self) { return self.dry_run_; },
                    [](ExecutionStrategy &self, bool dry_run) {
                      self.dry_run_ = dry_run;
                    });
C
chengduo 已提交
781

Y
yuyang18 已提交
782
  exec_strategy.def_property(
Y
yuyang18 已提交
783 784 785 786 787 788 789
      "use_experimental_executor",
      [](const ExecutionStrategy &self) {
        return self.type_ == ExecutionStrategy::kExperimental;
      },
      [](ExecutionStrategy &self, bool experimental) {
        self.type_ = experimental ? ExecutionStrategy::kExperimental
                                  : ExecutionStrategy::kDefault;
Y
yuyang18 已提交
790 791
      });

C
chengduo 已提交
792 793 794 795
  py::class_<BuildStrategy> build_strategy(pe, "BuildStrategy", R"DOC(
    BuildStrategy allows the user to more preciously control how to
    build the SSA Graph in ParallelExecutor by setting the property.

C
chengduo 已提交
796 797 798 799 800 801 802 803 804 805 806
    Examples:
        .. code-block:: python

          build_strategy = fluid.BuildStrategy()
          build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce

          train_exe = fluid.ParallelExecutor(use_cuda=True,
                                             loss_name=loss.name,
                                             build_strategy=build_strategy)

          train_loss, = train_exe.run([loss.name], feed=feed_dict)
C
chengduo 已提交
807
)DOC");
Y
yuyang18 已提交
808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823

  py::enum_<BuildStrategy::ReduceStrategy>(build_strategy, "ReduceStrategy")
      .value("Reduce", BuildStrategy::ReduceStrategy::kReduce)
      .value("AllReduce", BuildStrategy::ReduceStrategy::kAllReduce);
  py::enum_<BuildStrategy::GradientScaleStrategy>(build_strategy,
                                                  "GradientScaleStrategy")
      .value("CoeffNumDevice",
             BuildStrategy::GradientScaleStrategy::kCoeffNumDevice)
      .value("One", BuildStrategy::GradientScaleStrategy::kOne)
      .value("Customized", BuildStrategy::GradientScaleStrategy::kCustomized);

  build_strategy.def(py::init())
      .def_property(
          "reduce_strategy",
          [](const BuildStrategy &self) { return self.reduce_; },
          [](BuildStrategy &self, BuildStrategy::ReduceStrategy strategy) {
X
Xin Pan 已提交
824
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
Y
yuyang18 已提交
825
            self.reduce_ = strategy;
C
chengduo 已提交
826 827 828 829 830 831 832
          },
          R"DOC(The type is STR, there are two reduce strategies in ParallelExecutor,
                  'AllReduce' and 'Reduce'. If you want that all the parameters'
                  optimization are done on all devices independently, you should choose 'AllReduce';
                  if you choose 'Reduce', all the parameters' optimization will be evenly distributed
                  to different devices, and then broadcast the optimized parameter to other devices.
                  In some models, `Reduce` is faster. Default 'AllReduce'. )DOC")
Y
yuyang18 已提交
833 834 835 836 837
      .def_property(
          "gradient_scale_strategy",
          [](const BuildStrategy &self) { return self.gradient_scale_; },
          [](BuildStrategy &self,
             BuildStrategy::GradientScaleStrategy strategy) {
X
Xin Pan 已提交
838
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
Y
yuyang18 已提交
839
            self.gradient_scale_ = strategy;
C
chengduo 已提交
840 841 842 843 844 845
          },
          R"DOC(The type is STR, there are three ways of defining :math:`loss@grad` in
                   ParallelExecutor, 'CoeffNumDevice', 'One' and 'Customized'. By default,
                   ParallelExecutor sets the :math:`loss@grad` according to the number of devices.
                   If you want to customize :math:`loss@grad`, you can choose 'Customized'.
                   Default 'CoeffNumDevice'.)DOC")
Y
yuyang18 已提交
846 847 848 849
      .def_property(
          "debug_graphviz_path",
          [](const BuildStrategy &self) { return self.debug_graphviz_path_; },
          [](BuildStrategy &self, const std::string &path) {
X
Xin Pan 已提交
850
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
Y
yuyang18 已提交
851
            self.debug_graphviz_path_ = path;
C
chengduo 已提交
852 853 854 855
          },
          R"DOC(The type is STR, debug_graphviz_path indicate the path that
                    writing the SSA Graph to file in the form of graphviz, you.
                    It is useful for debugging. Default "")DOC")
F
fengjiayi 已提交
856 857 858
      .def_property(
          "enable_data_balance",
          [](const BuildStrategy &self) { return self.enable_data_balance_; },
C
chengduo 已提交
859
          [](BuildStrategy &self, bool b) {
X
Xin Pan 已提交
860
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
C
chengduo 已提交
861 862
            self.enable_data_balance_ = b;
          })  // FIXME(chengudo): enable_data_balance seems not important
S
sneaxiy 已提交
863 864 865 866 867 868
      .def_property(
          "enable_sequential_execution",
          [](const BuildStrategy &self) {
            return self.enable_sequential_execution_;
          },
          [](BuildStrategy &self, bool b) {
X
Xin Pan 已提交
869
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
S
sneaxiy 已提交
870 871 872 873 874 875 876 877 878
            self.enable_sequential_execution_ = b;
          },
          R"DOC(The type is BOOL. If set True, the execution order of ops would be the same as what is in the program. Default False.)DOC")
      .def_property(
          "remove_unnecessary_lock",
          [](const BuildStrategy &self) {
            return self.remove_unnecessary_lock_;
          },
          [](BuildStrategy &self, bool b) {
X
Xin Pan 已提交
879
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
S
sneaxiy 已提交
880 881 882
            self.remove_unnecessary_lock_ = b;
          },
          R"DOC(The type is BOOL. If set True, some locks in GPU ops would be released and ParallelExecutor would run faster. Default False.)DOC")
883 884 885 886 887 888
      .def_property(
          "num_trainers",
          [](const BuildStrategy &self) { return self.num_trainers_; },
          [](BuildStrategy &self, int num_trainers) {
            self.num_trainers_ = num_trainers;
          })
C
chengduo 已提交
889 890 891 892 893 894
      .def_property(
          "fuse_elewise_add_act_ops",
          [](const BuildStrategy &self) {
            return self.fuse_elewise_add_act_ops_;
          },
          [](BuildStrategy &self, bool b) {
X
Xin Pan 已提交
895
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
C
chengduo 已提交
896 897 898 899 900
            self.fuse_elewise_add_act_ops_ = b;
          },
          R"DOC(The type is BOOL, fuse_elewise_add_act_ops indicate whether
                     to fuse elementwise_add_op and activation_op,
                     it may make the execution faster. Default False)DOC")
901
      .def("_finalize_strategy_and_create_passes",
X
fix  
Xin Pan 已提交
902
           [](BuildStrategy &self) -> std::shared_ptr<ir::PassBuilder> {
903 904 905 906 907
             return self.CreatePassesFromStrategy(true);
           },
           R"DOC(Allow user to customized passes. Normally model-specific
                optimization passes should be defined in this way. BuildStrategy
                cannot be updated after being finalized.)DOC");
Y
yuyang18 已提交
908 909 910 911

  pe.def(py::init<const std::vector<platform::Place> &,
                  const std::unordered_set<std::string> &,
                  const std::unordered_set<std::string> &, const ProgramDesc &,
Y
yuyang18 已提交
912
                  const std::string &, Scope *, std::vector<Scope *> &,
913 914
                  const ExecutionStrategy &, const BuildStrategy &, size_t,
                  size_t>())
Y
Yu Yang 已提交
915 916 917 918
      // NOTE: even we return a vec<Scope*>* to Python use reference policy.
      // We still cannot get local_scope from this vector, since the element
      // of vec<Scope*> will be freed by Python GC. We can only return Scope*
      // one by one and mark them as reference.
919 920 921 922 923
      .def("local_scopes",
           [](ParallelExecutor &self) -> std::vector<Scope *> * {
             return &self.GetLocalScopes();
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
924 925 926 927
      .def("feed_tensors_into_local_scopes",
           &ParallelExecutor::FeedTensorsIntoLocalScopes)
      .def("feed_and_split_tensor_into_local_scopes",
           &ParallelExecutor::FeedAndSplitTensorIntoLocalScopes)
S
sneaxiy 已提交
928 929 930 931 932 933
      .def("run", [](ParallelExecutor &self,
                     const std::vector<std::string> &fetch_tensors,
                     const std::string &fetched_var_name) {
        pybind11::gil_scoped_release release;
        self.Run(fetch_tensors, fetched_var_name);
      });
Y
Yu Yang 已提交
934

935
  BindRecordIOWriter(&m);
W
Wang Guibao 已提交
936
  BindAsyncExecutor(&m);
L
Luo Tao 已提交
937
}
938
}  // namespace pybind
939
}  // namespace paddle