pybind.cc 39.6 KB
Newer Older
1
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
2 3 4 5 6

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

7
http://www.apache.org/licenses/LICENSE-2.0
8 9 10 11 12 13

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
L
lgone2000 已提交
14
#include <Python.h>
C
chengduoZH 已提交
15 16
#include <algorithm>
#include <map>
S
sneaxiy 已提交
17
#include <memory>
C
chengduoZH 已提交
18 19 20 21 22
#include <mutex>  // NOLINT // for call_once
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>
23

Y
Yi Wang 已提交
24 25 26
#include "paddle/fluid/framework/executor.h"
#include "paddle/fluid/framework/feed_fetch_method.h"
#include "paddle/fluid/framework/framework.pb.h"
27
#include "paddle/fluid/framework/ir/pass_builder.h"
Y
Yi Wang 已提交
28 29 30
#include "paddle/fluid/framework/lod_rank_table.h"
#include "paddle/fluid/framework/lod_tensor.h"
#include "paddle/fluid/framework/lod_tensor_array.h"
31
#include "paddle/fluid/framework/op_registry.h"
Y
Yu Yang 已提交
32
#include "paddle/fluid/framework/parallel_executor.h"
Y
Yi Wang 已提交
33
#include "paddle/fluid/framework/prune.h"
Y
Refine  
Yu Yang 已提交
34
#include "paddle/fluid/framework/reader.h"
Y
Yi Wang 已提交
35
#include "paddle/fluid/framework/selected_rows.h"
X
Xin Pan 已提交
36
#include "paddle/fluid/framework/version.h"
Y
Refine  
Yu Yang 已提交
37
#include "paddle/fluid/memory/allocation/allocator_strategy.h"
D
dzhwinter 已提交
38
#include "paddle/fluid/operators/activation_op.h"
S
sneaxiy 已提交
39
#include "paddle/fluid/operators/reader/lod_tensor_blocking_queue.h"
Y
Yu Yang 已提交
40
#include "paddle/fluid/platform/cpu_info.h"
Y
Yi Wang 已提交
41
#include "paddle/fluid/platform/enforce.h"
42
#include "paddle/fluid/platform/init.h"
Y
Yi Wang 已提交
43 44 45 46
#include "paddle/fluid/platform/place.h"
#include "paddle/fluid/platform/profiler.h"
#include "paddle/fluid/pybind/const_value.h"
#include "paddle/fluid/pybind/exception.h"
47 48
#include "paddle/fluid/pybind/protobuf.h"
#include "paddle/fluid/pybind/pybind.h"  // NOLINT
Y
Yu Yang 已提交
49
#include "paddle/fluid/pybind/recordio.h"
Y
Yi Wang 已提交
50
#include "paddle/fluid/pybind/tensor_py.h"
Y
Yu Yang 已提交
51

52
#include "paddle/fluid/string/to_string.h"
53

D
Dong Zhihong 已提交
54
#ifdef PADDLE_WITH_CUDA
P
peizhilin 已提交
55
#ifndef _WIN32
Y
Yi Wang 已提交
56
#include "paddle/fluid/operators/nccl/nccl_gpu_common.h"
P
peizhilin 已提交
57
#endif
Y
Yi Wang 已提交
58 59
#include "paddle/fluid/platform/cuda_profiler.h"
#include "paddle/fluid/platform/gpu_info.h"
D
Dong Zhihong 已提交
60 61
#endif

M
minqiyang 已提交
62 63
#include "pybind11/stl.h"

64 65 66 67
DEFINE_bool(reader_queue_speed_test_mode, false,
            "If set true, the queue.pop will only get data from queue but not "
            "remove the data from queue for speed testing");

Q
Qiao Longfei 已提交
68 69 70
// disable auto conversion to list in Python
PYBIND11_MAKE_OPAQUE(paddle::framework::LoDTensorArray);

71
namespace paddle {
72
namespace pybind {
73
bool IsCompiledWithCUDA() {
74
#ifndef PADDLE_WITH_CUDA
Q
qijun 已提交
75 76 77 78 79 80
  return false;
#else
  return true;
#endif
}

Y
update  
Yancey1989 已提交
81
bool IsCompiledWithDIST() {
Y
Yancey1989 已提交
82
#ifdef PADDLE_WITH_DISTRIBUTE
Y
update  
Yancey1989 已提交
83 84 85 86 87 88
  return true;
#else
  return false;
#endif
}

89
PYBIND11_MODULE(core, m) {
Y
Yu Yang 已提交
90 91 92
  // Not used, just make sure cpu_info.cc is linked.
  paddle::platform::CpuTotalPhysicalMemory();

Y
Refine  
Yu Yang 已提交
93
  paddle::memory::allocation::UseAllocatorStrategyGFlag();
94
  m.doc() = "C++ core of PaddlePaddle";
95

96 97 98 99
  // using framework in this function. Since it is inside a function, it will
  // not cause namespace pollution.
  using namespace paddle::framework;  // NOLINT

100
  BindException(&m);
Y
Yu Yang 已提交
101

102 103 104
  py::class_<Tensor>(m, "Tensor", py::buffer_protocol())
      .def_buffer(
          [](Tensor &self) -> py::buffer_info { return CastToPyBuffer(self); })
Y
yuyang18 已提交
105
      .def("_get_dims",
106
           [](const Tensor &self) { return vectorize(self.dims()); })
Y
yuyang18 已提交
107
      .def("_set_dims",
Q
qijun 已提交
108
           [](Tensor &self, const std::vector<int64_t> &dim) {
Y
Yu Yang 已提交
109
             self.Resize(make_ddim(dim));
Y
Yu Yang 已提交
110
           })
Y
yuyang18 已提交
111
      .def("_set_layout",
D
dzhwinter 已提交
112 113 114
           [](Tensor &self, const std::string &layout) {
             self.set_layout(StringToDataLayout(layout));
           })
Y
yuyang18 已提交
115
      .def("_alloc_float",
D
dzhwinter 已提交
116
           [](Tensor &self, paddle::platform::CUDAPlace &place) {
Q
qijun 已提交
117
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
118
           })
Y
yuyang18 已提交
119
      .def("_alloc_float",
Y
Yu Yang 已提交
120
           [](Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
121
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
122
           })
Y
yuyang18 已提交
123
      .def("_alloc_int",
Y
Yu Yang 已提交
124
           [](Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
125
             self.mutable_data<int>(place);
Y
Yu Yang 已提交
126
           })
Y
yuyang18 已提交
127
      .def("_alloc_int",
D
dzhwinter 已提交
128
           [](Tensor &self, paddle::platform::CUDAPlace &place) {
Q
qijun 已提交
129
             self.mutable_data<int>(place);
Q
qijun 已提交
130
           })
Y
yuyang18 已提交
131
      .def("_alloc_int",
C
chengduoZH 已提交
132 133 134
           [](Tensor &self, paddle::platform::CUDAPinnedPlace &place) {
             self.mutable_data<int>(place);
           })
Y
yuyang18 已提交
135
      .def("_alloc_float",
C
chengduoZH 已提交
136 137 138
           [](Tensor &self, paddle::platform::CUDAPinnedPlace &place) {
             self.mutable_data<float>(place);
           })
Y
Yu Yang 已提交
139 140
      .def("set", PyCPUTensorSetFromArray<float>)
      .def("set", PyCPUTensorSetFromArray<int>)
141
      .def("set", PyCPUTensorSetFromArray<double>)
142
      .def("set", PyCPUTensorSetFromArray<int64_t>)
Y
Yu Yang 已提交
143
      .def("set", PyCPUTensorSetFromArray<bool>)
144
      .def("set", PyCPUTensorSetFromArray<uint16_t>)
F
fengjiayi 已提交
145
      .def("set", PyCPUTensorSetFromArray<uint8_t>)
Q
qingqing01 已提交
146
      .def("set", PyCPUTensorSetFromArray<int8_t>)
147
#ifdef PADDLE_WITH_CUDA
Y
Yu Yang 已提交
148 149
      .def("set", PyCUDATensorSetFromArray<float>)
      .def("set", PyCUDATensorSetFromArray<int>)
150
      .def("set", PyCUDATensorSetFromArray<double>)
151
      .def("set", PyCUDATensorSetFromArray<int64_t>)
Y
Yu Yang 已提交
152
      .def("set", PyCUDATensorSetFromArray<bool>)
153
      .def("set", PyCUDATensorSetFromArray<uint16_t>)
F
fengjiayi 已提交
154
      .def("set", PyCUDATensorSetFromArray<uint8_t>)
Q
qingqing01 已提交
155
      .def("set", PyCUDATensorSetFromArray<int8_t>)
C
chengduoZH 已提交
156 157 158 159 160 161
      .def("set", PyCUDAPinnedTensorSetFromArray<float>)
      .def("set", PyCUDAPinnedTensorSetFromArray<int>)
      .def("set", PyCUDAPinnedTensorSetFromArray<double>)
      .def("set", PyCUDAPinnedTensorSetFromArray<int64_t>)
      .def("set", PyCUDAPinnedTensorSetFromArray<bool>)
      .def("set", PyCUDAPinnedTensorSetFromArray<uint16_t>)
F
fengjiayi 已提交
162
      .def("set", PyCUDAPinnedTensorSetFromArray<uint8_t>)
Q
qingqing01 已提交
163
      .def("set", PyCUDAPinnedTensorSetFromArray<int8_t>)
Q
qijun 已提交
164
#endif
165
      .def("shape", [](Tensor &self) { return vectorize(self.dims()); })
Y
yuyang18 已提交
166 167 168 169 170
      .def("_set_float_element", TensorSetElement<float>)
      .def("_get_float_element", TensorGetElement<float>)
      .def("_set_double_element", TensorSetElement<double>)
      .def("_get_double_element", TensorGetElement<double>)
      .def("_dtype", [](Tensor &self) { return ToDataType(self.type()); });
Y
Yu Yang 已提交
171

X
Xin Pan 已提交
172 173 174 175 176 177 178 179 180 181 182 183 184
  py::class_<LoDTensor, Tensor>(m, "LoDTensor", R"DOC(
    LoDTensor is a Tensor with optional LoD information.

    np.array(lod_tensor) can convert LoDTensor to numpy array.
    lod_tensor.lod() can retrieve the LoD information.

    LoD is short for Level of Details and is usually used for varied sequence
    length. You can skip the following comment if you don't need optional LoD.

  For example:
     A LoDTensor X can look like the example below. It contains 2 sequences.
     The first has length 2 and the second has length 3, as described by x.lod.

X
fix doc  
Xin Pan 已提交
185
     The first tensor dimension 5=2+3 is calculated from LoD if it's available.
X
Xin Pan 已提交
186
     It means the total number of sequence element. In X, each element has 2
X
fix doc  
Xin Pan 已提交
187
     columns, hence [5, 2].
X
Xin Pan 已提交
188 189 190

      x.lod  = [[2, 3]]
      x.data = [[1, 2], [3, 4],
X
fix doc  
Xin Pan 已提交
191 192
                [5, 6], [7, 8], [9, 10]]
      x.shape = [5, 2]
X
Xin Pan 已提交
193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215

      LoD can have multiple levels (for example, a paragraph can have multiple
      sentences and a sentence can have multiple words). In the following
      LodTensor Y, the lod_level is 2. It means there are 2 sequence, the
      first sequence length is 2 (has 2 sub-sequences), the second one's
      length is 1. The first sequence's 2 sub-sequences have length 2 and 2,
      respectively. And the second sequence's 1 sub-sequence has length 3.

      y.lod = [[2 1], [2 2 3]]
      y.shape = [2+2+3, ...]

  Note:
      In above description, LoD is length-based. In Paddle internal
      implementation, lod is offset-based. Hence, internally,
      y.lod is represented as [[0, 2, 3], [0, 2, 4, 7]] (length-based
      equivlent would be [[2-0, 3-2], [2-0, 4-2, 7-4]]).

      Sometimes LoD is called recursive_sequence_length to be more
      self-explanatory. In this case, it must be length-based. Due to history
      reasons. when LoD is called lod in public API, it might be offset-based.
      Users should be careful about it.

        )DOC")
216 217
      .def_buffer(
          [](Tensor &self) -> py::buffer_info { return CastToPyBuffer(self); })
218 219 220 221 222 223 224 225 226 227 228 229 230 231
      .def("__init__",
           [](LoDTensor &instance, const std::vector<std::vector<size_t>>
                                       &recursive_sequence_lengths) {
             LoD new_lod;
             new_lod.reserve(recursive_sequence_lengths.size());
             std::copy(recursive_sequence_lengths.begin(),
                       recursive_sequence_lengths.end(),
                       std::back_inserter(new_lod));
             LoD new_offset_lod = ConvertToOffsetBasedLoD(new_lod);
             PADDLE_ENFORCE(
                 CheckLoD(new_offset_lod, -1),
                 "the provided recursive_sequence_lengths info is invalid");
             new (&instance) LoDTensor(new_offset_lod);
           })
Y
Yu Yang 已提交
232
      .def("__init__", [](LoDTensor &instance) { new (&instance) LoDTensor(); })
G
gongweibao 已提交
233 234 235 236 237
      // We implement offset based LOD in C++ while we use length based with
      // Python API. So we changed set_lod to set_recursive_sequence_lengths to
      // avoid misuse.
      // The discussion is here:
      // https://github.com/PaddlePaddle/Paddle/issues/10855
D
dangqingqing 已提交
238
      .def("set_lod",
239
           [](LoDTensor &self, const std::vector<std::vector<size_t>> &lod) {
240
             // the input lod is offset-based level-of-detail info
Y
Yu Yang 已提交
241
             LoD new_lod;
242 243
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
244 245
             PADDLE_ENFORCE(CheckLoD(new_lod, vectorize(self.dims()).front()),
                            "the provided lod info is invalid");
246
             self.set_lod(new_lod);
D
dangqingqing 已提交
247
           })
248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272
      .def("set_recursive_sequence_lengths",
           [](LoDTensor &self, const std::vector<std::vector<size_t>>
                                   &recursive_sequence_lengths) {
             // the input recursive_sequence_lengths is length-based
             // level-of-detail info
             LoD new_lod;
             new_lod.reserve(recursive_sequence_lengths.size());
             std::copy(recursive_sequence_lengths.begin(),
                       recursive_sequence_lengths.end(),
                       std::back_inserter(new_lod));
             LoD new_offset_lod = ConvertToOffsetBasedLoD(new_lod);
             PADDLE_ENFORCE(
                 CheckLoD(new_offset_lod, vectorize(self.dims()).front()),
                 "the provided recursive_sequence_lengths info is invalid");
             self.set_lod(new_offset_lod);
           })
      .def("lod",
           [](LoDTensor &self) -> std::vector<std::vector<size_t>> {
             // output the offset-based lod info
             LoD lod = self.lod();
             std::vector<std::vector<size_t>> new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             return new_lod;
           })
G
gongweibao 已提交
273
      // Set above comments of set_lod.
274 275 276 277 278 279 280 281 282 283 284 285 286
      .def("recursive_sequence_lengths",
           [](LoDTensor &self) -> std::vector<std::vector<size_t>> {
             // output the length-based lod info
             LoD lod = ConvertToLengthBasedLoD(self.lod());
             std::vector<std::vector<size_t>> new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             return new_lod;
           })
      .def("has_valid_recursive_sequence_lengths", [](LoDTensor &self) -> bool {
        // Check that the lod info is valid and match the outermost
        // dimension of the LoDTensor data
        return CheckLoD(self.lod(), vectorize(self.dims()).front());
D
dangqingqing 已提交
287 288
      });

Q
qijun 已提交
289 290 291 292 293 294 295 296 297 298 299 300 301
  py::class_<SelectedRows>(m, "SelectedRows")
      .def("__init__",
           [](SelectedRows &instance) { new (&instance) SelectedRows(); })
      .def("__init__",
           [](SelectedRows &instance, const std::vector<int64_t> rows,
              const int64_t &height) {
             new (&instance) SelectedRows(rows, height);
           })
      .def("get_tensor",
           [](SelectedRows &self) { return self.mutable_value(); },
           py::return_value_policy::reference)
      .def("set_height", &SelectedRows::set_height)
      .def("height", &SelectedRows::height)
Q
qijun 已提交
302 303 304 305 306 307 308 309 310
      .def("set_rows",
           [](SelectedRows &self, std::vector<int64_t> rows) {
#ifndef PADDLE_WITH_CUDA
             self.set_rows(rows);
#else
        Vector<int64_t> new_rows(rows);
        self.set_rows(new_rows);
#endif
           })
311
      .def("sync_index", [](SelectedRows &instance) { instance.SyncIndex(); })
312
      .def("rows", [](SelectedRows &self) {
313 314 315 316 317
        auto rows = self.rows();
        std::vector<int64_t> new_rows;
        new_rows.reserve(rows.size());
        std::copy(rows.begin(), rows.end(), std::back_inserter(new_rows));
        return new_rows;
318
      });
Q
qijun 已提交
319

320
  py::class_<Variable>(m, "Variable", R"DOC(Variable Class.
321 322 323

All parameter, weight, gradient are variables in Paddle.
)DOC")
324
      .def("is_int", [](const Variable &var) { return var.IsType<int>(); })
325
      .def("set_int",
326 327
           [](Variable &var, int val) -> void { *var.GetMutable<int>() = val; })
      .def("get_int", [](const Variable &var) -> int { return var.Get<int>(); })
328 329 330 331 332 333 334
      .def("is_float", [](const Variable &var) { return var.IsType<float>(); })
      .def("set_float",
           [](Variable &var, float val) -> void {
             *var.GetMutable<float>() = val;
           })
      .def("get_float",
           [](const Variable &var) -> float { return var.Get<float>(); })
Y
Yu Yang 已提交
335
      .def("get_tensor",
336 337
           [](Variable &self) -> LoDTensor * {
             return self.GetMutable<LoDTensor>();
D
dangqingqing 已提交
338 339
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
340 341 342
      .def("get_lod_rank_table",
           [](Variable &self) { return self.GetMutable<LoDRankTable>(); },
           py::return_value_policy::reference)
Q
qijun 已提交
343 344 345 346 347
      .def("get_selected_rows",
           [](Variable &self) -> SelectedRows * {
             return self.GetMutable<SelectedRows>();
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
348 349 350
      .def("get_lod_tensor_array",
           [](Variable &self) { return self.GetMutable<LoDTensorArray>(); },
           py::return_value_policy::reference)
P
peizhilin 已提交
351
#if (defined(PADDLE_WITH_CUDA) && !defined(_WIN32))
D
Dong Zhihong 已提交
352 353 354 355 356
      .def("get_communicator",
           [](Variable &self) -> platform::Communicator * {
             return self.GetMutable<platform::Communicator>();
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
357
#endif
Y
Refine  
Yu Yang 已提交
358 359 360 361 362
      .def("get_reader",
           [](Variable &self) -> framework::ReaderHolder * {
             PADDLE_ENFORCE(self.IsType<framework::ReaderHolder>());
             return self.GetMutable<framework::ReaderHolder>();
           },
W
wopeizl 已提交
363
           py::return_value_policy::reference);
364

Y
Refine  
Yu Yang 已提交
365
  py::class_<framework::ReaderHolder>(m, "Reader", "")
366
      .def("reset", &framework::ReaderHolder::ResetAll);
Y
Refine  
Yu Yang 已提交
367

S
sneaxiy 已提交
368 369 370 371
  using LoDTensorBlockingQueue =
      ::paddle::operators::reader::LoDTensorBlockingQueue;
  using LoDTensorBlockingQueueHolder =
      ::paddle::operators::reader::LoDTensorBlockingQueueHolder;
S
sneaxiy 已提交
372 373
  py::class_<LoDTensorBlockingQueue, std::shared_ptr<LoDTensorBlockingQueue>>(
      m, "LoDTensorBlockingQueue", "")
S
sneaxiy 已提交
374
      .def("push",
S
sneaxiy 已提交
375
           [](LoDTensorBlockingQueue &self,
S
sneaxiy 已提交
376
              const std::vector<framework::LoDTensor> &lod_tensor_vec) {
S
sneaxiy 已提交
377
             pybind11::gil_scoped_release release;
S
sneaxiy 已提交
378
             return self.Push(lod_tensor_vec);
S
sneaxiy 已提交
379
           })
S
sneaxiy 已提交
380 381 382 383
      .def("size", &LoDTensorBlockingQueue::Size)
      .def("capacity", &LoDTensorBlockingQueue::Cap)
      .def("close", &LoDTensorBlockingQueue::Close)
      .def("is_closed", &LoDTensorBlockingQueue::IsClosed);
S
sneaxiy 已提交
384

S
sneaxiy 已提交
385
  m.def("init_lod_tensor_blocking_queue",
S
sneaxiy 已提交
386
        [](Variable &var, size_t capacity,
S
sneaxiy 已提交
387
           const std::vector<std::vector<int64_t>> &shapes)
S
sneaxiy 已提交
388
            -> std::shared_ptr<LoDTensorBlockingQueue> {
S
sneaxiy 已提交
389 390 391 392 393 394
              std::vector<DDim> dims(shapes.size());
              std::transform(shapes.begin(), shapes.end(), dims.begin(),
                             [](const std::vector<int64_t> &shape) {
                               return make_ddim(shape);
                             });
              auto *holder = var.GetMutable<LoDTensorBlockingQueueHolder>();
395 396
              holder->InitOnce(capacity, dims,
                               FLAGS_reader_queue_speed_test_mode);
S
sneaxiy 已提交
397
              return holder->GetQueue();
S
sneaxiy 已提交
398
            },
S
sneaxiy 已提交
399
        py::return_value_policy::copy);
S
sneaxiy 已提交
400

401
  py::class_<Scope>(m, "Scope", "")
D
dongzhihong 已提交
402
      .def("var",
403
           [](Scope &self, const std::string &name) -> Variable * {
D
dongzhihong 已提交
404
             return self.Var(name);
Y
Yu Yang 已提交
405
           },
406
           py::return_value_policy::reference)
407
      .def("find_var", &Scope::FindVar, py::return_value_policy::reference)
Y
Yu Yang 已提交
408
      .def(py::init<>())
409
      .def("new_scope", [](Scope &self) -> Scope * { return &self.NewScope(); },
410
           py::return_value_policy::reference)
Y
Yu Yang 已提交
411
      .def("drop_kids", &Scope::DropKids);
412

Y
Yu Yang 已提交
413 414
  //! @note: Be careful! PyBind will return std::string as an unicode, not
  //! Python str. If you want a str object, you should cast them in Python.
Y
Yu Yang 已提交
415 416
  m.def("get_all_op_protos", []() -> std::vector<py::bytes> {
    std::vector<py::bytes> ret_values;
417 418 419 420 421 422 423 424 425 426
    for (auto &iter : OpInfoMap::Instance().map()) {
      auto &info = iter.second;
      if (info.HasOpProtoAndChecker()) {
        std::string str;
        PADDLE_ENFORCE(
            info.Proto().SerializeToString(&str),
            "Serialize OpProto Error. This could be a bug of Paddle.");
        ret_values.emplace_back(str);
      }
    }
Y
Yu Yang 已提交
427 428
    return ret_values;
  });
429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444
  m.def(
      "get_grad_op_desc", [](const OpDesc &op_desc,
                             const std::unordered_set<std::string> &no_grad_set,
                             const std::vector<BlockDesc *> &grad_sub_block) {
        std::unordered_map<std::string, std::string> grad_to_var;
        std::vector<std::unique_ptr<OpDesc>> grad_op_descs =
            framework::OpInfoMap::Instance()
                .Get(op_desc.Type())
                .GradOpMaker()(op_desc, no_grad_set, &grad_to_var,
                               grad_sub_block);
        std::vector<OpDesc *> grad_op_desc_ptrs(grad_op_descs.size());
        std::transform(grad_op_descs.begin(), grad_op_descs.end(),
                       grad_op_desc_ptrs.begin(),
                       [](std::unique_ptr<OpDesc> &p) { return p.release(); });
        return std::make_pair(grad_op_desc_ptrs, grad_to_var);
      });
Y
Yu Yang 已提交
445
  m.def("prune", [](const ProgramDesc &origin,
446
                    const std::vector<std::array<size_t, 2>> &targets) {
Y
Yu Yang 已提交
447
    ProgramDesc prog_with_targets(origin);
448
    for (const auto &t : targets) {
449
      prog_with_targets.MutableBlock(t[0])->Op(t[1])->SetIsTarget(true);
450
    }
451
    proto::ProgramDesc pruned_desc;
452
    Prune(*prog_with_targets.Proto(), &pruned_desc);
Y
Yu Yang 已提交
453
    return new ProgramDesc(pruned_desc);
454
  });
455 456 457 458
  m.def("empty_var_name",
        []() { return std::string(framework::kEmptyVarName); });
  m.def("grad_var_suffix",
        []() { return std::string(framework::kGradVarSuffix); });
459 460 461
  m.def_submodule(
       "var_names",
       "The module will return special predefined variable name in Paddle")
Y
Yi Wang 已提交
462 463
      .def("empty", []() { return kEmptyVarName; })
      .def("temp", []() { return kTempVarName; });
Q
qijun 已提交
464
  // clang-format off
Y
Yu Yang 已提交
465
  py::class_<paddle::platform::DeviceContext>(m, "DeviceContext")
Q
qijun 已提交
466 467
      .def_static("create",
                  [](paddle::platform::CPUPlace& place)
Q
qijun 已提交
468
                      -> paddle::platform::DeviceContext* {
Q
qijun 已提交
469 470 471
                    return new paddle::platform::CPUDeviceContext();
                  })
      .def_static("create",
D
dzhwinter 已提交
472
                  [](paddle::platform::CUDAPlace& place)
Q
qijun 已提交
473
                      -> paddle::platform::DeviceContext* {
474
#ifndef PADDLE_WITH_CUDA
D
dzhwinter 已提交
475
                    PADDLE_THROW("CUDAPlace is not supported in CPU device.");
Q
qijun 已提交
476
#else
Q
qijun 已提交
477
                    return new paddle::platform::CUDADeviceContext(place);
Q
qijun 已提交
478
#endif
C
chengduoZH 已提交
479 480 481 482 483 484 485 486 487 488 489
                  })
          .def_static("create",
                [](paddle::platform::CUDAPinnedPlace& place)
                        -> paddle::platform::DeviceContext* {
#ifndef PADDLE_WITH_CUDA
                  PADDLE_THROW(
                        "CUDAPinnedPlace is not supported in CPU device.");
#else
                  return new paddle::platform::CUDAPinnedDeviceContext(place);
#endif
                });;
D
Dong Zhihong 已提交
490
// clang-format on
P
peizhilin 已提交
491
#if (defined(PADDLE_WITH_CUDA) && !defined(_WIN32))
D
Dong Zhihong 已提交
492 493
  py::class_<platform::Communicator>(m, "Communicator").def(py::init<>());
#endif
D
dzhwinter 已提交
494
  py::class_<platform::CUDAPlace>(m, "CUDAPlace")
495
      .def(py::init<int>())
D
dzhwinter 已提交
496
      .def("__str__", string::to_string<const platform::CUDAPlace &>);
Q
qijun 已提交
497

498 499 500
  py::class_<paddle::platform::CPUPlace>(m, "CPUPlace")
      .def(py::init<>())
      .def("__str__", string::to_string<const platform::CPUPlace &>);
Y
Yu Yang 已提交
501

C
chengduoZH 已提交
502 503 504 505
  py::class_<paddle::platform::CUDAPinnedPlace>(m, "CUDAPinnedPlace")
      .def(py::init<>())
      .def("__str__", string::to_string<const platform::CUDAPinnedPlace &>);

Y
Yu Yang 已提交
506 507 508 509 510 511 512
  py::class_<platform::Place>(m, "Place")
      .def(py::init<>())
      .def("set_place",
           [](platform::Place &self, const platform::CPUPlace &cpu_place) {
             self = cpu_place;
           })
      .def("set_place",
D
dzhwinter 已提交
513
           [](platform::Place &self, const platform::CUDAPlace &gpu_place) {
Y
Yu Yang 已提交
514
             self = gpu_place;
C
chengduoZH 已提交
515 516
           })
      .def("set_place", [](platform::Place &self,
C
chengduoZH 已提交
517 518
                           const platform::CUDAPinnedPlace &cuda_pinned_place) {
        self = cuda_pinned_place;
C
chengduoZH 已提交
519
      });
Y
Yu Yang 已提交
520

Y
Yu Yang 已提交
521 522 523
  py::class_<OperatorBase>(m, "Operator")
      .def_static("create",
                  [](py::bytes protobin) {
524
                    proto::OpDesc desc;
Y
Yu Yang 已提交
525 526 527 528 529
                    PADDLE_ENFORCE(desc.ParsePartialFromString(protobin),
                                   "Cannot parse user input to OpDesc");
                    PADDLE_ENFORCE(desc.IsInitialized(),
                                   "User OpDesc is not initialized, reason %s",
                                   desc.InitializationErrorString());
530
                    return OpRegistry::CreateOp(desc);
Y
Yu Yang 已提交
531
                  })
532
      .def("run",
533
           [](OperatorBase &self, const Scope &scope,
D
dzhwinter 已提交
534 535 536
              const platform::CPUPlace &place) { self.Run(scope, place); })
      .def("run",
           [](OperatorBase &self, const Scope &scope,
D
dzhwinter 已提交
537
              const platform::CUDAPlace &place) { self.Run(scope, place); })
C
chengduoZH 已提交
538 539 540 541 542
      .def("run",
           [](OperatorBase &self, const Scope &scope,
              const platform::CUDAPinnedPlace &place) {
             self.Run(scope, place);
           })
Y
Yu Yang 已提交
543 544 545 546 547 548 549
      .def("type",
           [](const OperatorBase &op) -> std::string { return op.Type(); })
      .def("outputs",
           [](const OperatorBase &op)
               -> std::map<std::string, std::vector<std::string>> {
                 return op.Outputs();
               })
Q
qijun 已提交
550 551
      .def("output_vars",
           [](const OperatorBase &op) { return op.OutputVars(true); })
Y
Yu Yang 已提交
552
      .def("inputs", [](const OperatorBase &op) { return op.Inputs(); })
Q
qijun 已提交
553
      .def("input_vars", [](const OperatorBase &op) { return op.InputVars(); })
Y
Yu Yang 已提交
554 555 556 557
      .def("__str__", &OperatorBase::DebugString)
      .def("no_intermediate_outputs",
           [](const OperatorBase &op) { return op.OutputVars(false); })
      .def("support_gpu", &OperatorBase::SupportGPU);
Y
Yu Yang 已提交
558

F
fengjiayi 已提交
559
  py::class_<framework::Executor>(m, "Executor")
D
dzhwinter 已提交
560
      .def(py::init<const platform::Place &>())
Y
Yancey1989 已提交
561
      .def("close", &Executor::Close)
S
sneaxiy 已提交
562 563 564 565 566
      .def("run", [](Executor &self, const ProgramDesc &prog, Scope *scope,
                     int block_id, bool create_local_scope, bool create_vars) {
        pybind11::gil_scoped_release release;
        self.Run(prog, scope, block_id, create_local_scope, create_vars);
      });
S
sneaxiy 已提交
567

D
dzhwinter 已提交
568
  m.def("init_gflags", framework::InitGflags);
Y
Yang Yu 已提交
569
  m.def("init_glog", framework::InitGLOG);
X
Xin Pan 已提交
570 571
  m.def("init_devices",
        [](bool init_p2p) { framework::InitDevices(init_p2p); });
572

573
  m.def("is_compiled_with_cuda", IsCompiledWithCUDA);
Y
update  
Yancey1989 已提交
574
  m.def("is_compiled_with_dist", IsCompiledWithDIST);
575 576 577 578 579 580
#ifdef PADDLE_WITH_CUDA
  m.def("is_float16_supported", [](const platform::CUDAPlace &place) -> bool {
    // Only GPUs with Compute Capability >= 53 support float16
    return platform::GetCUDAComputeCapability(place.device) >= 53;
  });
#endif
581

582
  m.def("set_feed_variable", framework::SetFeedVariable);
Q
qijun 已提交
583
  m.def("get_fetch_variable", framework::GetFetchVariable);
Q
qijun 已提交
584

X
Xin Pan 已提交
585 586
  m.def("_is_program_version_supported", IsProgramVersionSupported);

587 588 589 590 591
  BindProgramDesc(&m);
  BindBlockDesc(&m);
  BindVarDsec(&m);
  BindOpDesc(&m);
  BindConstValue(&m);
Y
Yu Yang 已提交
592

Y
Yu Yang 已提交
593 594 595 596 597 598 599 600 601
  py::class_<framework::LoDRankTable>(m, "LodRankTable")
      .def("items", [](framework::LoDRankTable &table) {
        std::vector<std::pair<size_t, size_t>> res;
        for (auto &item : table.items()) {
          res.push_back({item.index, item.length});
        }
        return res;
      });

Y
Yu Yang 已提交
602
  py::class_<LoDTensorArray>(m, "LoDTensorArray")
S
sneaxiy 已提交
603 604
      .def("__init__",
           [](LoDTensorArray &instance) { new (&instance) LoDTensorArray(); })
Y
Yu Yang 已提交
605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620
      .def("__getitem__",
           [](LoDTensorArray &self, size_t i) { return &self.at(i); },
           py::return_value_policy::reference)
      .def("__len__", [](LoDTensorArray &self) { return self.size(); })
      .def("__setitem__",
           [](LoDTensorArray &self, size_t i, const LoDTensor &t) {
             PADDLE_ENFORCE_LT(i, self.size());
             self[i].ShareDataWith(t);
             self[i].set_lod(t.lod());
           })
      .def("append", [](LoDTensorArray &self, const LoDTensor &t) {
        self.emplace_back();
        self.back().ShareDataWith(t);
        self.back().set_lod(t.lod());
      });

D
dzhwinter 已提交
621 622 623
  m.def("IsInplace",
        [](std::string op) -> bool { return operators::IsInplace(op); });

Y
Yu Yang 已提交
624
  m.def("op_support_gpu", OpSupportGPU);
D
Dong Zhihong 已提交
625
#ifdef PADDLE_WITH_CUDA
D
Dong Zhihong 已提交
626
  m.def("get_cuda_device_count", platform::GetCUDADeviceCount);
D
dangqingqing 已提交
627

P
peizhilin 已提交
628
#ifndef _WIN32
D
dangqingqing 已提交
629 630 631
  m.def("nvprof_init", platform::CudaProfilerInit);
  m.def("nvprof_start", platform::CudaProfilerStart);
  m.def("nvprof_stop", platform::CudaProfilerStop);
D
Dong Zhihong 已提交
632
#endif
P
peizhilin 已提交
633
#endif
Y
Yu Yang 已提交
634

635 636 637 638
  py::enum_<platform::ProfilerState>(m, "ProfilerState", py::arithmetic())
      .value("kDisabled", platform::ProfilerState::kDisabled)
      .value("kCPU", platform::ProfilerState::kCPU)
      .value("kCUDA", platform::ProfilerState::kCUDA)
639
      .value("kAll", platform::ProfilerState::kAll)
640 641 642 643 644 645 646 647 648 649 650 651 652
      .export_values();

  py::enum_<platform::EventSortingKey>(m, "EventSortingKey", py::arithmetic())
      .value("kDefault", platform::EventSortingKey::kDefault)
      .value("kCalls", platform::EventSortingKey::kCalls)
      .value("kTotal", platform::EventSortingKey::kTotal)
      .value("kMin", platform::EventSortingKey::kMin)
      .value("kMax", platform::EventSortingKey::kMax)
      .value("kAve", platform::EventSortingKey::kAve)
      .export_values();

  m.def("enable_profiler", platform::EnableProfiler);
  m.def("disable_profiler", platform::DisableProfiler);
X
Xin Pan 已提交
653
  m.def("is_profiler_enabled", platform::IsProfileEnabled);
654
  m.def("reset_profiler", platform::ResetProfiler);
Y
Yu Yang 已提交
655

656 657
  py::class_<ir::Pass, std::shared_ptr<ir::Pass>> pass(m, "Pass");
  pass.def(py::init())
658 659 660 661 662
      .def(
          "set_str",
          [](ir::Pass &self, const std::string &name, const std::string &attr) {
            self.Set<std::string>(name, new std::string(attr));
          })
X
Xin Pan 已提交
663 664 665
      .def("set_int", [](ir::Pass &self, const std::string &name,
                         int val) { self.Set<const int>(name, new int(val)); })
      .def("type", &ir::Pass::Type);
666

X
fix  
Xin Pan 已提交
667 668
  py::class_<ir::PassBuilder, std::shared_ptr<ir::PassBuilder>> pb(
      m, "PassBuilder");
669 670 671 672 673 674 675 676 677 678 679 680 681 682
  pb.def(py::init())
      .def("append_pass",
           [](ir::PassBuilder &self,
              const std::string &pass_type) -> std::shared_ptr<ir::Pass> {
             return self.AppendPass(pass_type);
           })
      .def("all_passes", [](ir::PassBuilder &self) { return self.AllPasses(); })
      .def("insert_pass",
           [](ir::PassBuilder &self, size_t idx, const std::string &pass_type) {
             return self.InsertPass(idx, pass_type);
           })
      .def("remove_pass",
           [](ir::PassBuilder &self, size_t idx) { self.RemovePass(idx); });

Y
yuyang18 已提交
683
  // -- python binds for parallel executor.
Y
yuyang18 已提交
684
  py::class_<ParallelExecutor> pe(m, "ParallelExecutor");
C
chengduo 已提交
685 686 687 688
  py::class_<ExecutionStrategy> exec_strategy(pe, "ExecutionStrategy", R"DOC(
    ExecutionStrategy allows the user to more preciously control how to run
    the program in ParallelExecutor by setting the property.

C
chengduo 已提交
689 690 691 692 693 694 695 696 697 698 699
    Examples:
        .. code-block:: python

          exec_strategy = fluid.ExecutionStrategy()
          exec_strategy.num_threads = 4

          train_exe = fluid.ParallelExecutor(use_cuda=True,
                                             loss_name=loss.name,
                                             exec_strategy=exec_strategy)

          train_loss, = train_exe.run([loss.name], feed=feed_dict)
C
chengduo 已提交
700 701 702

        )DOC");

Y
yuyang18 已提交
703
  exec_strategy.def(py::init())
Y
yuyang18 已提交
704 705 706 707 708
      .def_property(
          "num_threads",
          [](const ExecutionStrategy &self) { return self.num_threads_; },
          [](ExecutionStrategy &self, size_t num_threads) {
            self.num_threads_ = num_threads;
C
chengduo 已提交
709 710 711 712 713 714 715 716 717 718
          },
          R"DOC(The type is INT, num_threads represents the size of thread pool that
            used to run the operators of the current program in ParallelExecutor.
            If :math:`num\_threads=1`, all the operators will execute one by one,
            but the order maybe difference between iterations.
            If it is not set, it will be set in ParallelExecutor according to the
            device type and device count, for GPU, :math:`num\_threads=device\_count*4`, for CPU,
            :math:`num\_threads=CPU\_NUM*4`, the explanation of:math:`CPU\_NUM` is in ParallelExecutor.
            if it is not set, ParallelExecutor will get the cpu count by calling
            `multiprocessing.cpu_count()`. Default 0.)DOC")
Y
yuyang18 已提交
719
      .def_property(
720 721 722 723
          "use_cuda",
          [](const ExecutionStrategy &self) { return self.use_cuda_; },
          [](ExecutionStrategy &self, bool use_cuda) {
            self.use_cuda_ = use_cuda;
C
chengduo 已提交
724 725 726 727
          })  // FIXME(chengduo): Doesn't add doc for 'use_cuda', use_cuda may
      // make user confuse, because ParallelExecutor has a parameter named
      // 'use_cuda' too, in current implementation, ParallelExecutor's
      // 'use_cuda' will rewrite ExecutionStrategy's 'use_cuda'.
Y
yuyang18 已提交
728 729 730 731 732
      .def_property(
          "allow_op_delay",
          [](const ExecutionStrategy &self) { return self.allow_op_delay_; },
          [](ExecutionStrategy &self, bool allow_op_delay) {
            self.allow_op_delay_ = allow_op_delay;
C
chengduo 已提交
733 734 735 736
          },
          R"DOC(The type is BOOL, allow_op_delay represents whether to delay the
                communication operators to run, it may make the execution faster.
                Note that in some models, allow_op_delay may cause program hang. Default False.)DOC")
Y
yuyang18 已提交
737 738 739 740 741 742 743
      .def_property(
          "num_iteration_per_drop_scope",
          [](const ExecutionStrategy &self) {
            return self.num_iteration_per_drop_scope_;
          },
          [](ExecutionStrategy &self, size_t num_iteration_per_drop_scope) {
            self.num_iteration_per_drop_scope_ = num_iteration_per_drop_scope;
C
chengduo 已提交
744 745 746 747 748 749 750 751 752 753 754
          },
          R"DOC(The type is INT, num_iteration_per_drop_scope indicates how
                many iterations to clean up the temp variables which
                is generated during execution. It may make the execution faster,
                because the temp variable's shape maybe the same between two iterations. Default 100.

                NOTES:
                    1. If you fetch data when calling the 'run', the ParallelExecutor
                       will clean up the temp variables at the end of the current iteration.
                    2. In some NLP model, it may cause the GPU memory is insufficient,
                       in this case, you should reduce `num_iteration_per_drop_scope`.
755 756 757 758 759 760
              )DOC")
      .def_property("_dry_run",
                    [](const ExecutionStrategy &self) { return self.dry_run_; },
                    [](ExecutionStrategy &self, bool dry_run) {
                      self.dry_run_ = dry_run;
                    });
C
chengduo 已提交
761

Y
yuyang18 已提交
762
  exec_strategy.def_property(
Y
yuyang18 已提交
763 764 765 766 767 768 769
      "use_experimental_executor",
      [](const ExecutionStrategy &self) {
        return self.type_ == ExecutionStrategy::kExperimental;
      },
      [](ExecutionStrategy &self, bool experimental) {
        self.type_ = experimental ? ExecutionStrategy::kExperimental
                                  : ExecutionStrategy::kDefault;
Y
yuyang18 已提交
770 771
      });

C
chengduo 已提交
772 773 774 775
  py::class_<BuildStrategy> build_strategy(pe, "BuildStrategy", R"DOC(
    BuildStrategy allows the user to more preciously control how to
    build the SSA Graph in ParallelExecutor by setting the property.

C
chengduo 已提交
776 777 778 779 780 781 782 783 784 785 786
    Examples:
        .. code-block:: python

          build_strategy = fluid.BuildStrategy()
          build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce

          train_exe = fluid.ParallelExecutor(use_cuda=True,
                                             loss_name=loss.name,
                                             build_strategy=build_strategy)

          train_loss, = train_exe.run([loss.name], feed=feed_dict)
C
chengduo 已提交
787
)DOC");
Y
yuyang18 已提交
788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803

  py::enum_<BuildStrategy::ReduceStrategy>(build_strategy, "ReduceStrategy")
      .value("Reduce", BuildStrategy::ReduceStrategy::kReduce)
      .value("AllReduce", BuildStrategy::ReduceStrategy::kAllReduce);
  py::enum_<BuildStrategy::GradientScaleStrategy>(build_strategy,
                                                  "GradientScaleStrategy")
      .value("CoeffNumDevice",
             BuildStrategy::GradientScaleStrategy::kCoeffNumDevice)
      .value("One", BuildStrategy::GradientScaleStrategy::kOne)
      .value("Customized", BuildStrategy::GradientScaleStrategy::kCustomized);

  build_strategy.def(py::init())
      .def_property(
          "reduce_strategy",
          [](const BuildStrategy &self) { return self.reduce_; },
          [](BuildStrategy &self, BuildStrategy::ReduceStrategy strategy) {
X
Xin Pan 已提交
804
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
Y
yuyang18 已提交
805
            self.reduce_ = strategy;
C
chengduo 已提交
806 807 808 809 810 811 812
          },
          R"DOC(The type is STR, there are two reduce strategies in ParallelExecutor,
                  'AllReduce' and 'Reduce'. If you want that all the parameters'
                  optimization are done on all devices independently, you should choose 'AllReduce';
                  if you choose 'Reduce', all the parameters' optimization will be evenly distributed
                  to different devices, and then broadcast the optimized parameter to other devices.
                  In some models, `Reduce` is faster. Default 'AllReduce'. )DOC")
Y
yuyang18 已提交
813 814 815 816 817
      .def_property(
          "gradient_scale_strategy",
          [](const BuildStrategy &self) { return self.gradient_scale_; },
          [](BuildStrategy &self,
             BuildStrategy::GradientScaleStrategy strategy) {
X
Xin Pan 已提交
818
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
Y
yuyang18 已提交
819
            self.gradient_scale_ = strategy;
C
chengduo 已提交
820 821 822 823 824 825
          },
          R"DOC(The type is STR, there are three ways of defining :math:`loss@grad` in
                   ParallelExecutor, 'CoeffNumDevice', 'One' and 'Customized'. By default,
                   ParallelExecutor sets the :math:`loss@grad` according to the number of devices.
                   If you want to customize :math:`loss@grad`, you can choose 'Customized'.
                   Default 'CoeffNumDevice'.)DOC")
Y
yuyang18 已提交
826 827 828 829
      .def_property(
          "debug_graphviz_path",
          [](const BuildStrategy &self) { return self.debug_graphviz_path_; },
          [](BuildStrategy &self, const std::string &path) {
X
Xin Pan 已提交
830
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
Y
yuyang18 已提交
831
            self.debug_graphviz_path_ = path;
C
chengduo 已提交
832 833 834 835
          },
          R"DOC(The type is STR, debug_graphviz_path indicate the path that
                    writing the SSA Graph to file in the form of graphviz, you.
                    It is useful for debugging. Default "")DOC")
F
fengjiayi 已提交
836 837 838
      .def_property(
          "enable_data_balance",
          [](const BuildStrategy &self) { return self.enable_data_balance_; },
C
chengduo 已提交
839
          [](BuildStrategy &self, bool b) {
X
Xin Pan 已提交
840
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
C
chengduo 已提交
841 842
            self.enable_data_balance_ = b;
          })  // FIXME(chengudo): enable_data_balance seems not important
S
sneaxiy 已提交
843 844 845 846 847 848
      .def_property(
          "enable_sequential_execution",
          [](const BuildStrategy &self) {
            return self.enable_sequential_execution_;
          },
          [](BuildStrategy &self, bool b) {
X
Xin Pan 已提交
849
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
S
sneaxiy 已提交
850 851 852 853 854 855 856 857 858
            self.enable_sequential_execution_ = b;
          },
          R"DOC(The type is BOOL. If set True, the execution order of ops would be the same as what is in the program. Default False.)DOC")
      .def_property(
          "remove_unnecessary_lock",
          [](const BuildStrategy &self) {
            return self.remove_unnecessary_lock_;
          },
          [](BuildStrategy &self, bool b) {
X
Xin Pan 已提交
859
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
S
sneaxiy 已提交
860 861 862
            self.remove_unnecessary_lock_ = b;
          },
          R"DOC(The type is BOOL. If set True, some locks in GPU ops would be released and ParallelExecutor would run faster. Default False.)DOC")
863 864 865 866 867 868
      .def_property(
          "num_trainers",
          [](const BuildStrategy &self) { return self.num_trainers_; },
          [](BuildStrategy &self, int num_trainers) {
            self.num_trainers_ = num_trainers;
          })
C
chengduo 已提交
869 870 871 872 873 874
      .def_property(
          "fuse_elewise_add_act_ops",
          [](const BuildStrategy &self) {
            return self.fuse_elewise_add_act_ops_;
          },
          [](BuildStrategy &self, bool b) {
X
Xin Pan 已提交
875
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
C
chengduo 已提交
876 877 878 879 880
            self.fuse_elewise_add_act_ops_ = b;
          },
          R"DOC(The type is BOOL, fuse_elewise_add_act_ops indicate whether
                     to fuse elementwise_add_op and activation_op,
                     it may make the execution faster. Default False)DOC")
881
      .def("_finalize_strategy_and_create_passes",
X
fix  
Xin Pan 已提交
882
           [](BuildStrategy &self) -> std::shared_ptr<ir::PassBuilder> {
883 884 885 886 887
             return self.CreatePassesFromStrategy(true);
           },
           R"DOC(Allow user to customized passes. Normally model-specific
                optimization passes should be defined in this way. BuildStrategy
                cannot be updated after being finalized.)DOC");
Y
yuyang18 已提交
888 889 890 891

  pe.def(py::init<const std::vector<platform::Place> &,
                  const std::unordered_set<std::string> &,
                  const std::unordered_set<std::string> &, const ProgramDesc &,
Y
yuyang18 已提交
892
                  const std::string &, Scope *, std::vector<Scope *> &,
893 894
                  const ExecutionStrategy &, const BuildStrategy &, size_t,
                  size_t>())
Y
Yu Yang 已提交
895 896 897 898
      // NOTE: even we return a vec<Scope*>* to Python use reference policy.
      // We still cannot get local_scope from this vector, since the element
      // of vec<Scope*> will be freed by Python GC. We can only return Scope*
      // one by one and mark them as reference.
899 900 901 902 903
      .def("local_scopes",
           [](ParallelExecutor &self) -> std::vector<Scope *> * {
             return &self.GetLocalScopes();
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
904 905 906 907
      .def("feed_tensors_into_local_scopes",
           &ParallelExecutor::FeedTensorsIntoLocalScopes)
      .def("feed_and_split_tensor_into_local_scopes",
           &ParallelExecutor::FeedAndSplitTensorIntoLocalScopes)
S
sneaxiy 已提交
908 909 910 911 912 913
      .def("run", [](ParallelExecutor &self,
                     const std::vector<std::string> &fetch_tensors,
                     const std::string &fetched_var_name) {
        pybind11::gil_scoped_release release;
        self.Run(fetch_tensors, fetched_var_name);
      });
Y
Yu Yang 已提交
914

915
  BindRecordIOWriter(&m);
L
Luo Tao 已提交
916
}
917
}  // namespace pybind
918
}  // namespace paddle