pybind.cc 52.8 KB
Newer Older
1
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
2 3 4 5 6

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

7
http://www.apache.org/licenses/LICENSE-2.0
8 9 10 11 12 13

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
L
lgone2000 已提交
14
#include <Python.h>
C
chengduoZH 已提交
15 16
#include <algorithm>
#include <map>
S
sneaxiy 已提交
17
#include <memory>
C
chengduoZH 已提交
18 19 20 21 22
#include <mutex>  // NOLINT // for call_once
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>
23

Y
Yi Wang 已提交
24 25 26
#include "paddle/fluid/framework/executor.h"
#include "paddle/fluid/framework/feed_fetch_method.h"
#include "paddle/fluid/framework/framework.pb.h"
27
#include "paddle/fluid/framework/ir/pass_builder.h"
Y
Yi Wang 已提交
28 29 30
#include "paddle/fluid/framework/lod_rank_table.h"
#include "paddle/fluid/framework/lod_tensor.h"
#include "paddle/fluid/framework/lod_tensor_array.h"
31
#include "paddle/fluid/framework/op_registry.h"
Y
Yu Yang 已提交
32
#include "paddle/fluid/framework/parallel_executor.h"
Y
Yi Wang 已提交
33
#include "paddle/fluid/framework/prune.h"
Y
Refine  
Yu Yang 已提交
34
#include "paddle/fluid/framework/reader.h"
S
sneaxiy 已提交
35
#include "paddle/fluid/framework/scope_pool.h"
Y
Yi Wang 已提交
36
#include "paddle/fluid/framework/selected_rows.h"
X
Xin Pan 已提交
37
#include "paddle/fluid/framework/version.h"
38
#include "paddle/fluid/imperative/layer.h"
Y
Refine  
Yu Yang 已提交
39
#include "paddle/fluid/memory/allocation/allocator_strategy.h"
40
#include "paddle/fluid/memory/allocation/legacy_allocator.h"
D
dzhwinter 已提交
41
#include "paddle/fluid/operators/activation_op.h"
S
sneaxiy 已提交
42
#include "paddle/fluid/operators/py_func_op.h"
S
sneaxiy 已提交
43
#include "paddle/fluid/operators/reader/lod_tensor_blocking_queue.h"
Y
Yu Yang 已提交
44
#include "paddle/fluid/platform/cpu_info.h"
Y
Yi Wang 已提交
45
#include "paddle/fluid/platform/enforce.h"
46
#include "paddle/fluid/platform/init.h"
Y
Yi Wang 已提交
47 48
#include "paddle/fluid/platform/place.h"
#include "paddle/fluid/platform/profiler.h"
W
Wang Guibao 已提交
49
#include "paddle/fluid/pybind/async_executor_py.h"
Y
Yi Wang 已提交
50 51
#include "paddle/fluid/pybind/const_value.h"
#include "paddle/fluid/pybind/exception.h"
52
#include "paddle/fluid/pybind/imperative.h"
F
flame 已提交
53
#include "paddle/fluid/pybind/inference_api.h"
F
flame 已提交
54
#include "paddle/fluid/pybind/ir.h"
55 56
#include "paddle/fluid/pybind/protobuf.h"
#include "paddle/fluid/pybind/pybind.h"  // NOLINT
Y
Yu Yang 已提交
57
#include "paddle/fluid/pybind/recordio.h"
Y
Yi Wang 已提交
58
#include "paddle/fluid/pybind/tensor_py.h"
Y
Yu Yang 已提交
59

60
#include "paddle/fluid/string/to_string.h"
61

D
Dong Zhihong 已提交
62
#ifdef PADDLE_WITH_CUDA
P
peizhilin 已提交
63
#ifndef _WIN32
Y
Yi Wang 已提交
64
#include "paddle/fluid/operators/nccl/nccl_gpu_common.h"
P
peizhilin 已提交
65
#endif
Y
Yi Wang 已提交
66 67
#include "paddle/fluid/platform/cuda_profiler.h"
#include "paddle/fluid/platform/gpu_info.h"
D
Dong Zhihong 已提交
68 69
#endif

M
minqiyang 已提交
70 71
#include "pybind11/stl.h"

72 73 74 75
DEFINE_bool(reader_queue_speed_test_mode, false,
            "If set true, the queue.pop will only get data from queue but not "
            "remove the data from queue for speed testing");

Q
Qiao Longfei 已提交
76 77 78
// disable auto conversion to list in Python
PYBIND11_MAKE_OPAQUE(paddle::framework::LoDTensorArray);

79
namespace paddle {
80
namespace pybind {
81
bool IsCompiledWithCUDA() {
82
#ifndef PADDLE_WITH_CUDA
Q
qijun 已提交
83 84 85 86 87 88
  return false;
#else
  return true;
#endif
}

89
bool IsCompiledWithBrpc() {
90
#ifndef PADDLE_WITH_DISTRIBUTE
91 92
  return false;
#endif
93 94 95 96 97 98

#ifdef PADDLE_WITH_GRPC
  return false;
#endif

  return true;
99 100
}

Y
update  
Yancey1989 已提交
101
bool IsCompiledWithDIST() {
Y
Yancey1989 已提交
102
#ifdef PADDLE_WITH_DISTRIBUTE
Y
update  
Yancey1989 已提交
103 104 105 106 107 108
  return true;
#else
  return false;
#endif
}

S
sneaxiy 已提交
109 110 111 112 113
template <typename PlaceType1, typename PlaceType2>
static inline bool IsSamePlace(const PlaceType1 &p1, const PlaceType2 &p2) {
  return paddle::platform::Place(p1) == paddle::platform::Place(p2);
}

114
PYBIND11_MODULE(core, m) {
Y
Yu Yang 已提交
115 116 117
  // Not used, just make sure cpu_info.cc is linked.
  paddle::platform::CpuTotalPhysicalMemory();

Y
Refine  
Yu Yang 已提交
118
  paddle::memory::allocation::UseAllocatorStrategyGFlag();
119
  m.doc() = "C++ core of PaddlePaddle";
120

121 122 123 124
  // using framework in this function. Since it is inside a function, it will
  // not cause namespace pollution.
  using namespace paddle::framework;  // NOLINT

125
  BindException(&m);
Y
Yu Yang 已提交
126

S
sneaxiy 已提交
127
  m.def(
S
sneaxiy 已提交
128
      "_append_python_callable_object_and_return_id",
S
sneaxiy 已提交
129 130 131 132
      [](py::object py_obj) -> size_t {
        return paddle::operators::AppendPythonCallableObjectAndReturnId(py_obj);
      });

S
sneaxiy 已提交
133 134 135
  m.add_object("_cleanup",
               py::capsule([]() { ScopePool::Instance().Clear(); }));

136 137 138 139 140 141 142
  m.def("get_mem_usage", [](int device) {
    return memory::allocation::GPUMemMonitor.GetMemUsage(device);
  });

  m.def("print_mem_usage",
        []() { return memory::allocation::GPUMemMonitor.PrintMemUsage(); });

M
minqiyang 已提交
143
  py::class_<imperative::VarBase>(m, "VarBase", R"DOC()DOC")
144 145
      // .def(py::init<>())
      .def(py::init<bool>(), py::arg("stop_gradient") = false)
146
      .def("_run_backward",
X
Xin Pan 已提交
147
           [](imperative::VarBase &self) { self.RunBackward(); })
M
minqiyang 已提交
148
      .def("_grad_name", &imperative::VarBase::GradName)
M
minqiyang 已提交
149
      .def("_grad_value", &imperative::VarBase::GradValue)
X
Xin Pan 已提交
150
      .def("_clear_gradient", &imperative::VarBase::ClearGradient)
M
minqiyang 已提交
151
      .def("_grad_ivar",
M
minqiyang 已提交
152
           [](const imperative::VarBase &self) { return self.grads_; },
M
minqiyang 已提交
153
           py::return_value_policy::reference)
M
minqiyang 已提交
154
      .def("_copy_to",
P
Paddle CI 已提交
155
           [](const imperative::VarBase &self, const platform::CPUPlace &place,
M
minqiyang 已提交
156 157 158 159 160
              bool blocking) {
             std::unique_ptr<imperative::VarBase> new_var =
                 self.NewVarBase(place, blocking);
             return new_var.release();
           },
P
Paddle CI 已提交
161
           py::return_value_policy::take_ownership)
M
minqiyang 已提交
162
      .def("_copy_to",
P
Paddle CI 已提交
163
           [](const imperative::VarBase &self, const platform::CUDAPlace &place,
M
minqiyang 已提交
164 165 166 167 168
              bool blocking) {
             std::unique_ptr<imperative::VarBase> new_var =
                 self.NewVarBase(place, blocking);
             return new_var.release();
           },
M
minqiyang 已提交
169
           py::return_value_policy::take_ownership)
M
minqiyang 已提交
170
      .def("value", [](const imperative::VarBase &self) { return self.var_; },
M
minqiyang 已提交
171
           py::return_value_policy::reference)
172 173 174 175 176 177
      .def_property(
          "desc",
          [](const imperative::VarBase &self) { return self.var_desc_; },
          [](imperative::VarBase &self, framework::VarDesc *var_desc) {
            self.var_desc_ = var_desc;
          },
178 179 180
          py::return_value_policy::reference)
      .def_property(
          "stop_gradient",
X
Xin Pan 已提交
181
          [](const imperative::VarBase &self) { return self.IsStopGradient(); },
182
          [](imperative::VarBase &self, bool stop_gradient) {
X
Xin Pan 已提交
183
            self.SetStopGradient(stop_gradient);
184
          });
185

186
  py::class_<imperative::OpBase, PyOpBase>(m, "OpBase", R"DOC()DOC")
187 188 189 190 191 192 193 194
      .def(py::init<>())
      .def_property(
          "desc", [](const imperative::OpBase &self) { return self.op_desc_; },
          [](imperative::OpBase &self, framework::OpDesc *op_desc) {
            if (op_desc) {
              self.op_desc_ = op_desc;
            }
          },
X
Xin Pan 已提交
195 196 197 198 199 200 201
          py::return_value_policy::reference)
      .def_property(
          "forward_id",
          [](const imperative::OpBase &self) { return self.forward_id_; },
          [](imperative::OpBase &self, int forward_id) {
            self.forward_id_ = forward_id;
          },
X
Xin Pan 已提交
202 203 204 205 206 207 208
          py::return_value_policy::reference)
      .def_property(
          "backward_id",
          [](const imperative::OpBase &self) { return self.backward_id_; },
          [](imperative::OpBase &self, int backward_id) {
            self.backward_id_ = backward_id;
          },
209 210
          py::return_value_policy::reference);

X
Xin Pan 已提交
211
  py::class_<imperative::Layer, Layer /* <--- trampoline*/> layer(m, "Layer");
212
  layer.def(py::init<>())
X
Xin Pan 已提交
213 214 215
      .def("forward", [](imperative::Layer &self,
                         const std::vector<imperative::VarBase> &inputs) {
        return self.Forward(inputs);
X
Xin Pan 已提交
216
      });
X
Xin Pan 已提交
217

X
polish  
Xin Pan 已提交
218
  py::class_<imperative::PyLayer>(m, "PyLayer")
X
Xin Pan 已提交
219
      .def(py::init<>())
X
Xin Pan 已提交
220 221
      .def_static(
          "apply",
X
Xin Pan 已提交
222
          [](int func_id, const std::vector<imperative::VarBase *> &inputs)
X
Xin Pan 已提交
223 224 225 226
              -> std::vector<imperative::VarBase *> {
                return imperative::PyLayer::Apply(func_id, inputs);
              },
          py::return_value_policy::take_ownership)
X
polish  
Xin Pan 已提交
227 228 229 230 231
      .def_static("register_func",
                  [](int func_id, const py::object &callable) {
                    imperative::PyLayer::RegisterFunc(func_id, callable);
                  })
      .def_static("num_funcs", &imperative::PyLayer::NumFuncs);
X
Xin Pan 已提交
232

233 234
  BindTracer(&m);

235 236 237
  py::class_<Tensor>(m, "Tensor", py::buffer_protocol())
      .def_buffer(
          [](Tensor &self) -> py::buffer_info { return CastToPyBuffer(self); })
Y
yuyang18 已提交
238
      .def("_get_dims",
239
           [](const Tensor &self) { return vectorize(self.dims()); })
Y
yuyang18 已提交
240
      .def("_set_dims",
Q
qijun 已提交
241
           [](Tensor &self, const std::vector<int64_t> &dim) {
Y
Yu Yang 已提交
242
             self.Resize(make_ddim(dim));
Y
Yu Yang 已提交
243
           })
Y
yuyang18 已提交
244
      .def("_set_layout",
D
dzhwinter 已提交
245 246 247
           [](Tensor &self, const std::string &layout) {
             self.set_layout(StringToDataLayout(layout));
           })
Y
yuyang18 已提交
248
      .def("_alloc_float",
D
dzhwinter 已提交
249
           [](Tensor &self, paddle::platform::CUDAPlace &place) {
Q
qijun 已提交
250
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
251
           })
Y
yuyang18 已提交
252
      .def("_alloc_float",
Y
Yu Yang 已提交
253
           [](Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
254
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
255
           })
Y
yuyang18 已提交
256
      .def("_alloc_int",
Y
Yu Yang 已提交
257
           [](Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
258
             self.mutable_data<int>(place);
Y
Yu Yang 已提交
259
           })
Y
yuyang18 已提交
260
      .def("_alloc_int",
D
dzhwinter 已提交
261
           [](Tensor &self, paddle::platform::CUDAPlace &place) {
Q
qijun 已提交
262
             self.mutable_data<int>(place);
Q
qijun 已提交
263
           })
Y
yuyang18 已提交
264
      .def("_alloc_int",
C
chengduoZH 已提交
265 266 267
           [](Tensor &self, paddle::platform::CUDAPinnedPlace &place) {
             self.mutable_data<int>(place);
           })
Y
yuyang18 已提交
268
      .def("_alloc_float",
C
chengduoZH 已提交
269 270 271
           [](Tensor &self, paddle::platform::CUDAPinnedPlace &place) {
             self.mutable_data<float>(place);
           })
Y
Yu Yang 已提交
272 273
      .def("set", PyCPUTensorSetFromArray<float>)
      .def("set", PyCPUTensorSetFromArray<int>)
274
      .def("set", PyCPUTensorSetFromArray<double>)
275
      .def("set", PyCPUTensorSetFromArray<int64_t>)
Y
Yu Yang 已提交
276
      .def("set", PyCPUTensorSetFromArray<bool>)
277
      .def("set", PyCPUTensorSetFromArray<uint16_t>)
F
fengjiayi 已提交
278
      .def("set", PyCPUTensorSetFromArray<uint8_t>)
Q
qingqing01 已提交
279
      .def("set", PyCPUTensorSetFromArray<int8_t>)
280
#ifdef PADDLE_WITH_CUDA
Y
Yu Yang 已提交
281 282
      .def("set", PyCUDATensorSetFromArray<float>)
      .def("set", PyCUDATensorSetFromArray<int>)
283
      .def("set", PyCUDATensorSetFromArray<double>)
284
      .def("set", PyCUDATensorSetFromArray<int64_t>)
Y
Yu Yang 已提交
285
      .def("set", PyCUDATensorSetFromArray<bool>)
286
      .def("set", PyCUDATensorSetFromArray<uint16_t>)
F
fengjiayi 已提交
287
      .def("set", PyCUDATensorSetFromArray<uint8_t>)
Q
qingqing01 已提交
288
      .def("set", PyCUDATensorSetFromArray<int8_t>)
C
chengduoZH 已提交
289 290 291 292 293 294
      .def("set", PyCUDAPinnedTensorSetFromArray<float>)
      .def("set", PyCUDAPinnedTensorSetFromArray<int>)
      .def("set", PyCUDAPinnedTensorSetFromArray<double>)
      .def("set", PyCUDAPinnedTensorSetFromArray<int64_t>)
      .def("set", PyCUDAPinnedTensorSetFromArray<bool>)
      .def("set", PyCUDAPinnedTensorSetFromArray<uint16_t>)
F
fengjiayi 已提交
295
      .def("set", PyCUDAPinnedTensorSetFromArray<uint8_t>)
Q
qingqing01 已提交
296
      .def("set", PyCUDAPinnedTensorSetFromArray<int8_t>)
Q
qijun 已提交
297
#endif
298
      .def("shape", [](Tensor &self) { return vectorize(self.dims()); })
Y
yuyang18 已提交
299 300 301 302
      .def("_set_float_element", TensorSetElement<float>)
      .def("_get_float_element", TensorGetElement<float>)
      .def("_set_double_element", TensorSetElement<double>)
      .def("_get_double_element", TensorGetElement<double>)
X
xuezhong 已提交
303
      .def("_place", [](Tensor &self) { return self.place(); })
Y
Yu Yang 已提交
304
      .def("_dtype", [](Tensor &self) { return self.type(); });
Y
Yu Yang 已提交
305

X
Xin Pan 已提交
306 307 308 309 310 311 312 313 314 315 316 317 318
  py::class_<LoDTensor, Tensor>(m, "LoDTensor", R"DOC(
    LoDTensor is a Tensor with optional LoD information.

    np.array(lod_tensor) can convert LoDTensor to numpy array.
    lod_tensor.lod() can retrieve the LoD information.

    LoD is short for Level of Details and is usually used for varied sequence
    length. You can skip the following comment if you don't need optional LoD.

  For example:
     A LoDTensor X can look like the example below. It contains 2 sequences.
     The first has length 2 and the second has length 3, as described by x.lod.

X
fix doc  
Xin Pan 已提交
319
     The first tensor dimension 5=2+3 is calculated from LoD if it's available.
X
Xin Pan 已提交
320
     It means the total number of sequence element. In X, each element has 2
X
fix doc  
Xin Pan 已提交
321
     columns, hence [5, 2].
X
Xin Pan 已提交
322 323 324

      x.lod  = [[2, 3]]
      x.data = [[1, 2], [3, 4],
X
fix doc  
Xin Pan 已提交
325 326
                [5, 6], [7, 8], [9, 10]]
      x.shape = [5, 2]
X
Xin Pan 已提交
327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349

      LoD can have multiple levels (for example, a paragraph can have multiple
      sentences and a sentence can have multiple words). In the following
      LodTensor Y, the lod_level is 2. It means there are 2 sequence, the
      first sequence length is 2 (has 2 sub-sequences), the second one's
      length is 1. The first sequence's 2 sub-sequences have length 2 and 2,
      respectively. And the second sequence's 1 sub-sequence has length 3.

      y.lod = [[2 1], [2 2 3]]
      y.shape = [2+2+3, ...]

  Note:
      In above description, LoD is length-based. In Paddle internal
      implementation, lod is offset-based. Hence, internally,
      y.lod is represented as [[0, 2, 3], [0, 2, 4, 7]] (length-based
      equivlent would be [[2-0, 3-2], [2-0, 4-2, 7-4]]).

      Sometimes LoD is called recursive_sequence_length to be more
      self-explanatory. In this case, it must be length-based. Due to history
      reasons. when LoD is called lod in public API, it might be offset-based.
      Users should be careful about it.

        )DOC")
350 351
      .def_buffer(
          [](Tensor &self) -> py::buffer_info { return CastToPyBuffer(self); })
352 353 354 355 356 357 358 359 360 361 362 363 364 365
      .def("__init__",
           [](LoDTensor &instance, const std::vector<std::vector<size_t>>
                                       &recursive_sequence_lengths) {
             LoD new_lod;
             new_lod.reserve(recursive_sequence_lengths.size());
             std::copy(recursive_sequence_lengths.begin(),
                       recursive_sequence_lengths.end(),
                       std::back_inserter(new_lod));
             LoD new_offset_lod = ConvertToOffsetBasedLoD(new_lod);
             PADDLE_ENFORCE(
                 CheckLoD(new_offset_lod, -1),
                 "the provided recursive_sequence_lengths info is invalid");
             new (&instance) LoDTensor(new_offset_lod);
           })
Y
Yu Yang 已提交
366
      .def("__init__", [](LoDTensor &instance) { new (&instance) LoDTensor(); })
G
gongweibao 已提交
367 368 369 370 371
      // We implement offset based LOD in C++ while we use length based with
      // Python API. So we changed set_lod to set_recursive_sequence_lengths to
      // avoid misuse.
      // The discussion is here:
      // https://github.com/PaddlePaddle/Paddle/issues/10855
D
dangqingqing 已提交
372
      .def("set_lod",
373
           [](LoDTensor &self, const std::vector<std::vector<size_t>> &lod) {
374
             // the input lod is offset-based level-of-detail info
Y
Yu Yang 已提交
375
             LoD new_lod;
376 377
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
378 379
             PADDLE_ENFORCE(CheckLoD(new_lod, vectorize(self.dims()).front()),
                            "the provided lod info is invalid");
380
             self.set_lod(new_lod);
S
sneaxiy 已提交
381 382 383 384 385 386 387
           },
           py::arg("lod"), R"DOC(
           Set LoD of the LoDTensor.

           Args:
               lod (List[List[int]]): the lod to be set.
           )DOC")
388 389 390 391 392 393 394 395 396 397 398 399 400 401 402
      .def("set_recursive_sequence_lengths",
           [](LoDTensor &self, const std::vector<std::vector<size_t>>
                                   &recursive_sequence_lengths) {
             // the input recursive_sequence_lengths is length-based
             // level-of-detail info
             LoD new_lod;
             new_lod.reserve(recursive_sequence_lengths.size());
             std::copy(recursive_sequence_lengths.begin(),
                       recursive_sequence_lengths.end(),
                       std::back_inserter(new_lod));
             LoD new_offset_lod = ConvertToOffsetBasedLoD(new_lod);
             PADDLE_ENFORCE(
                 CheckLoD(new_offset_lod, vectorize(self.dims()).front()),
                 "the provided recursive_sequence_lengths info is invalid");
             self.set_lod(new_offset_lod);
S
sneaxiy 已提交
403 404 405 406
           },
           py::arg("recursive_sequence_lengths"), R"DOC(
           Set LoD of the LoDTensor according to recursive sequence length.

S
sneaxiy 已提交
407
           For example, if recursive_sequence_lengths=[[2, 3]], meaning that
S
sneaxiy 已提交
408
           there are two sequences with length 2 and 3 respectively, the 
S
sneaxiy 已提交
409
           corresponding lod would be [[0, 2, 2+3]], i.e, [[0, 2, 5]].  
S
sneaxiy 已提交
410 411 412 413

           Args:
                recursive_sequence_lengths (List[List[int]]): sequence lengths. 
           )DOC")
414 415 416 417 418 419 420 421
      .def("lod",
           [](LoDTensor &self) -> std::vector<std::vector<size_t>> {
             // output the offset-based lod info
             LoD lod = self.lod();
             std::vector<std::vector<size_t>> new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             return new_lod;
S
sneaxiy 已提交
422 423 424 425 426 427 428
           },
           R"DOC(
           Return the LoD of the LoDTensor.

           Returns:
               out (List[List[int]]): the lod of the LoDTensor.
           )DOC")
G
gongweibao 已提交
429
      // Set above comments of set_lod.
430 431 432 433 434 435 436 437
      .def("recursive_sequence_lengths",
           [](LoDTensor &self) -> std::vector<std::vector<size_t>> {
             // output the length-based lod info
             LoD lod = ConvertToLengthBasedLoD(self.lod());
             std::vector<std::vector<size_t>> new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             return new_lod;
S
sneaxiy 已提交
438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456
           },
           R"DOC(
           Return the sequence length of the LoDTensor corresponding to LoD.

           Returns:
               out (List[List[int]): the sequence lengths. 
           )DOC")
      .def("has_valid_recursive_sequence_lengths",
           [](LoDTensor &self) -> bool {
             // Check that the lod info is valid and match the outermost
             // dimension of the LoDTensor data
             return CheckLoD(self.lod(), vectorize(self.dims()).front());
           },
           R"DOC(
           Check whether the lod of the LoDTensor is valid.

           Returns:
               out (bool): whether the lod is valid.
           )DOC");
D
dangqingqing 已提交
457

Q
qijun 已提交
458 459 460 461 462 463 464 465 466 467 468
  py::class_<SelectedRows>(m, "SelectedRows")
      .def("__init__",
           [](SelectedRows &instance) { new (&instance) SelectedRows(); })
      .def("__init__",
           [](SelectedRows &instance, const std::vector<int64_t> rows,
              const int64_t &height) {
             new (&instance) SelectedRows(rows, height);
           })
      .def("get_tensor",
           [](SelectedRows &self) { return self.mutable_value(); },
           py::return_value_policy::reference)
469 470
      .def("numel",
           [](SelectedRows &self) -> int64_t { return self.value().numel(); })
Q
qijun 已提交
471 472
      .def("set_height", &SelectedRows::set_height)
      .def("height", &SelectedRows::height)
Q
qijun 已提交
473 474 475 476 477 478 479 480 481
      .def("set_rows",
           [](SelectedRows &self, std::vector<int64_t> rows) {
#ifndef PADDLE_WITH_CUDA
             self.set_rows(rows);
#else
        Vector<int64_t> new_rows(rows);
        self.set_rows(new_rows);
#endif
           })
482
      .def("sync_index", [](SelectedRows &instance) { instance.SyncIndex(); })
483
      .def("rows", [](SelectedRows &self) {
484 485 486 487 488
        auto rows = self.rows();
        std::vector<int64_t> new_rows;
        new_rows.reserve(rows.size());
        std::copy(rows.begin(), rows.end(), std::back_inserter(new_rows));
        return new_rows;
489
      });
Q
qijun 已提交
490

491
  py::class_<Variable>(m, "Variable", R"DOC(Variable Class.
492 493 494

All parameter, weight, gradient are variables in Paddle.
)DOC")
495
      .def("is_int", [](const Variable &var) { return var.IsType<int>(); })
496
      .def("set_int",
497 498
           [](Variable &var, int val) -> void { *var.GetMutable<int>() = val; })
      .def("get_int", [](const Variable &var) -> int { return var.Get<int>(); })
499 500 501 502 503 504 505
      .def("is_float", [](const Variable &var) { return var.IsType<float>(); })
      .def("set_float",
           [](Variable &var, float val) -> void {
             *var.GetMutable<float>() = val;
           })
      .def("get_float",
           [](const Variable &var) -> float { return var.Get<float>(); })
Y
Yu Yang 已提交
506
      .def("get_tensor",
507 508
           [](Variable &self) -> LoDTensor * {
             return self.GetMutable<LoDTensor>();
D
dangqingqing 已提交
509 510
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
511 512 513
      .def("get_lod_rank_table",
           [](Variable &self) { return self.GetMutable<LoDRankTable>(); },
           py::return_value_policy::reference)
Q
qijun 已提交
514 515 516 517 518
      .def("get_selected_rows",
           [](Variable &self) -> SelectedRows * {
             return self.GetMutable<SelectedRows>();
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
519 520 521
      .def("get_lod_tensor_array",
           [](Variable &self) { return self.GetMutable<LoDTensorArray>(); },
           py::return_value_policy::reference)
P
peizhilin 已提交
522
#if (defined(PADDLE_WITH_CUDA) && !defined(_WIN32))
D
Dong Zhihong 已提交
523 524 525 526 527
      .def("get_communicator",
           [](Variable &self) -> platform::Communicator * {
             return self.GetMutable<platform::Communicator>();
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
528
#endif
Y
Refine  
Yu Yang 已提交
529 530 531 532 533
      .def("get_reader",
           [](Variable &self) -> framework::ReaderHolder * {
             PADDLE_ENFORCE(self.IsType<framework::ReaderHolder>());
             return self.GetMutable<framework::ReaderHolder>();
           },
W
wopeizl 已提交
534
           py::return_value_policy::reference);
535

Y
Refine  
Yu Yang 已提交
536
  py::class_<framework::ReaderHolder>(m, "Reader", "")
Q
Qiao Longfei 已提交
537
      .def("start", &framework::ReaderHolder::Start)
538
      .def("reset", &framework::ReaderHolder::ResetAll);
Y
Refine  
Yu Yang 已提交
539

S
sneaxiy 已提交
540 541 542 543
  using LoDTensorBlockingQueue =
      ::paddle::operators::reader::LoDTensorBlockingQueue;
  using LoDTensorBlockingQueueHolder =
      ::paddle::operators::reader::LoDTensorBlockingQueueHolder;
S
sneaxiy 已提交
544 545
  py::class_<LoDTensorBlockingQueue, std::shared_ptr<LoDTensorBlockingQueue>>(
      m, "LoDTensorBlockingQueue", "")
S
sneaxiy 已提交
546
      .def("push",
S
sneaxiy 已提交
547
           [](LoDTensorBlockingQueue &self,
S
sneaxiy 已提交
548
              const std::vector<framework::LoDTensor> &lod_tensor_vec) {
S
sneaxiy 已提交
549
             pybind11::gil_scoped_release release;
S
sneaxiy 已提交
550
             return self.Push(lod_tensor_vec);
S
sneaxiy 已提交
551
           })
S
sneaxiy 已提交
552 553 554 555
      .def("size", &LoDTensorBlockingQueue::Size)
      .def("capacity", &LoDTensorBlockingQueue::Cap)
      .def("close", &LoDTensorBlockingQueue::Close)
      .def("is_closed", &LoDTensorBlockingQueue::IsClosed);
S
sneaxiy 已提交
556

S
sneaxiy 已提交
557
  m.def("init_lod_tensor_blocking_queue",
Q
Qiao Longfei 已提交
558 559 560 561 562 563
        [](Variable &var,
           size_t capacity) -> std::shared_ptr<LoDTensorBlockingQueue> {
          auto *holder = var.GetMutable<LoDTensorBlockingQueueHolder>();
          holder->InitOnce(capacity, FLAGS_reader_queue_speed_test_mode);
          return holder->GetQueue();
        },
S
sneaxiy 已提交
564
        py::return_value_policy::copy);
S
sneaxiy 已提交
565

S
sneaxiy 已提交
566
  py::class_<Scope>(m, "_Scope", R"DOC(
Q
Qiao Longfei 已提交
567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585
    Scope is an association of a name to Variable. All variables belong to Scope.

    Variables in a parent scope can be retrieved from local scope.

    You need to specify a scope to run a Net, i.e., `exe.Run(&scope)`.
    One net can run in different scopes and update different variable in the
    scope.

    You can create var in a scope and get it from the scope.

    Examples:
        .. code-block:: python

          # create tensor from a scope and set value to it.
          param = scope.var('Param').get_tensor()
          param_array = np.full((height, row_numel), 5.0).astype("float32")
          param.set(param_array, place)

        )DOC")
S
sneaxiy 已提交
586 587
      .def("_remove_from_pool",
           [](Scope &self) { ScopePool::Instance().Remove(&self); })
D
dongzhihong 已提交
588
      .def("var",
589
           [](Scope &self, const std::string &name) -> Variable * {
D
dongzhihong 已提交
590
             return self.Var(name);
Y
Yu Yang 已提交
591
           },
S
sneaxiy 已提交
592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617
           py::arg("name"),
           R"DOC(
           Find or create variable named :code:`name` in the current scope. 

           If the variable named :code:`name` does not exist in the 
           current scope, the variable would be created. Otherwise,
           return the existing variable. 

           Args:
               name (str): the variable name.  
          
           Returns:
               out (core.Variable): the found or created variable. 
           )DOC",
           py::return_value_policy::reference)
      .def("find_var", &Scope::FindVar, py::arg("name"),
           R"DOC(
           Find variable named :code:`name` in the current scope or 
           its parent scope. Return None if not found.
        
           Args:
               name (str): the variable name.
            
           Returns:
               out (core.Variable|None): the found variable or None.   
           )DOC",
618
           py::return_value_policy::reference)
619
      .def("new_scope", [](Scope &self) -> Scope * { return &self.NewScope(); },
S
sneaxiy 已提交
620 621 622 623 624 625
           R"DOC(
           Create a new sub-scope of the current scope.

           Returns:
               out (core._Scope): the created sub-scope.
           )DOC",
626
           py::return_value_policy::reference)
S
sneaxiy 已提交
627 628 629 630
      .def("drop_kids", &Scope::DropKids,
           R"DOC(
           Delete all sub-scopes of the current scope.
           )DOC");
631

S
sneaxiy 已提交
632 633 634 635 636 637
  m.def("Scope",
        []() -> Scope * {
          auto *s = new Scope();
          ScopePool::Instance().Insert(std::unique_ptr<Scope>(s));
          return s;
        },
S
sneaxiy 已提交
638 639 640 641 642 643
        R"DOC(
        Create a new scope.
        
        Returns:
            out (core._Scope): the created scope.
        )DOC",
S
sneaxiy 已提交
644 645
        py::return_value_policy::reference);

Y
Yu Yang 已提交
646 647
  //! @note: Be careful! PyBind will return std::string as an unicode, not
  //! Python str. If you want a str object, you should cast them in Python.
Y
Yu Yang 已提交
648 649
  m.def("get_all_op_protos", []() -> std::vector<py::bytes> {
    std::vector<py::bytes> ret_values;
650 651 652 653 654 655 656 657 658 659
    for (auto &iter : OpInfoMap::Instance().map()) {
      auto &info = iter.second;
      if (info.HasOpProtoAndChecker()) {
        std::string str;
        PADDLE_ENFORCE(
            info.Proto().SerializeToString(&str),
            "Serialize OpProto Error. This could be a bug of Paddle.");
        ret_values.emplace_back(str);
      }
    }
Y
Yu Yang 已提交
660 661
    return ret_values;
  });
662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677
  m.def(
      "get_grad_op_desc", [](const OpDesc &op_desc,
                             const std::unordered_set<std::string> &no_grad_set,
                             const std::vector<BlockDesc *> &grad_sub_block) {
        std::unordered_map<std::string, std::string> grad_to_var;
        std::vector<std::unique_ptr<OpDesc>> grad_op_descs =
            framework::OpInfoMap::Instance()
                .Get(op_desc.Type())
                .GradOpMaker()(op_desc, no_grad_set, &grad_to_var,
                               grad_sub_block);
        std::vector<OpDesc *> grad_op_desc_ptrs(grad_op_descs.size());
        std::transform(grad_op_descs.begin(), grad_op_descs.end(),
                       grad_op_desc_ptrs.begin(),
                       [](std::unique_ptr<OpDesc> &p) { return p.release(); });
        return std::make_pair(grad_op_desc_ptrs, grad_to_var);
      });
Y
Yu Yang 已提交
678
  m.def("prune", [](const ProgramDesc &origin,
679
                    const std::vector<std::array<size_t, 2>> &targets) {
Y
Yu Yang 已提交
680
    ProgramDesc prog_with_targets(origin);
681
    for (const auto &t : targets) {
682
      prog_with_targets.MutableBlock(t[0])->Op(t[1])->SetIsTarget(true);
683
    }
684
    proto::ProgramDesc pruned_desc;
685
    Prune(*prog_with_targets.Proto(), &pruned_desc);
Y
Yu Yang 已提交
686
    return new ProgramDesc(pruned_desc);
687
  });
688 689 690 691
  m.def("empty_var_name",
        []() { return std::string(framework::kEmptyVarName); });
  m.def("grad_var_suffix",
        []() { return std::string(framework::kGradVarSuffix); });
692 693 694
  m.def_submodule(
       "var_names",
       "The module will return special predefined variable name in Paddle")
Y
Yi Wang 已提交
695 696
      .def("empty", []() { return kEmptyVarName; })
      .def("temp", []() { return kTempVarName; });
Q
qijun 已提交
697
  // clang-format off
Y
Yu Yang 已提交
698
  py::class_<paddle::platform::DeviceContext>(m, "DeviceContext")
Q
qijun 已提交
699 700
      .def_static("create",
                  [](paddle::platform::CPUPlace& place)
Q
qijun 已提交
701
                      -> paddle::platform::DeviceContext* {
Q
qijun 已提交
702 703 704
                    return new paddle::platform::CPUDeviceContext();
                  })
      .def_static("create",
D
dzhwinter 已提交
705
                  [](paddle::platform::CUDAPlace& place)
Q
qijun 已提交
706
                      -> paddle::platform::DeviceContext* {
707
#ifndef PADDLE_WITH_CUDA
D
dzhwinter 已提交
708
                    PADDLE_THROW("CUDAPlace is not supported in CPU device.");
Q
qijun 已提交
709
#else
Q
qijun 已提交
710
                    return new paddle::platform::CUDADeviceContext(place);
Q
qijun 已提交
711
#endif
C
chengduoZH 已提交
712 713 714 715 716 717 718 719 720 721 722
                  })
          .def_static("create",
                [](paddle::platform::CUDAPinnedPlace& place)
                        -> paddle::platform::DeviceContext* {
#ifndef PADDLE_WITH_CUDA
                  PADDLE_THROW(
                        "CUDAPinnedPlace is not supported in CPU device.");
#else
                  return new paddle::platform::CUDAPinnedDeviceContext(place);
#endif
                });;
D
Dong Zhihong 已提交
723
// clang-format on
P
peizhilin 已提交
724
#if (defined(PADDLE_WITH_CUDA) && !defined(_WIN32))
D
Dong Zhihong 已提交
725 726
  py::class_<platform::Communicator>(m, "Communicator").def(py::init<>());
#endif
D
dzhwinter 已提交
727
  py::class_<platform::CUDAPlace>(m, "CUDAPlace")
S
sneaxiy 已提交
728 729 730 731 732 733 734 735 736 737 738 739
      .def("__init__",
           [](platform::CUDAPlace &self, int dev_id) {
#ifdef PADDLE_WITH_CUDA
             PADDLE_ENFORCE(
                 dev_id >= 0 && dev_id < platform::GetCUDADeviceCount(),
                 "Invalid CUDAPlace(%d), must inside [0, %d)", dev_id,
                 platform::GetCUDADeviceCount());
             new (&self) platform::CUDAPlace(dev_id);
#else
             PADDLE_THROW("Cannot use CUDAPlace in CPU only version");
#endif
           })
S
sneaxiy 已提交
740 741 742 743 744
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::Place>)
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::CPUPlace>)
      .def("_equals",
           &IsSamePlace<platform::CUDAPlace, platform::CUDAPinnedPlace>)
D
dzhwinter 已提交
745
      .def("__str__", string::to_string<const platform::CUDAPlace &>);
Q
qijun 已提交
746

747 748
  py::class_<paddle::platform::CPUPlace>(m, "CPUPlace")
      .def(py::init<>())
S
sneaxiy 已提交
749 750 751 752 753
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::Place>)
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::CPUPlace>)
      .def("_equals",
           &IsSamePlace<platform::CPUPlace, platform::CUDAPinnedPlace>)
754
      .def("__str__", string::to_string<const platform::CPUPlace &>);
Y
Yu Yang 已提交
755

C
chengduoZH 已提交
756
  py::class_<paddle::platform::CUDAPinnedPlace>(m, "CUDAPinnedPlace")
S
sneaxiy 已提交
757
      .def("__init__",
S
sneaxiy 已提交
758
           [](platform::CUDAPinnedPlace &self) {
S
sneaxiy 已提交
759 760 761
#ifndef PADDLE_WITH_CUDA
             PADDLE_THROW("Cannot use CUDAPinnedPlace in CPU only version");
#endif
S
sneaxiy 已提交
762
             new (&self) platform::CUDAPinnedPlace();
S
sneaxiy 已提交
763
           })
S
sneaxiy 已提交
764 765 766 767 768 769 770
      .def("_equals", &IsSamePlace<platform::CUDAPinnedPlace, platform::Place>)
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::CUDAPlace>)
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::CPUPlace>)
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::CUDAPinnedPlace>)
C
chengduoZH 已提交
771 772
      .def("__str__", string::to_string<const platform::CUDAPinnedPlace &>);

Y
Yu Yang 已提交
773 774
  py::class_<platform::Place>(m, "Place")
      .def(py::init<>())
S
sneaxiy 已提交
775 776 777 778
      .def("_equals", &IsSamePlace<platform::Place, platform::Place>)
      .def("_equals", &IsSamePlace<platform::Place, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::Place, platform::CPUPlace>)
      .def("_equals", &IsSamePlace<platform::Place, platform::CUDAPinnedPlace>)
X
xuezhong 已提交
779 780 781 782 783 784
      .def("is_gpu_place",
           [](platform::Place &self) { return platform::is_gpu_place(self); })
      .def("gpu_device_id",
           [](platform::Place &self) {
             return boost::get<platform::CUDAPlace>(self).device;
           })
Y
Yu Yang 已提交
785 786 787 788 789
      .def("set_place",
           [](platform::Place &self, const platform::CPUPlace &cpu_place) {
             self = cpu_place;
           })
      .def("set_place",
D
dzhwinter 已提交
790
           [](platform::Place &self, const platform::CUDAPlace &gpu_place) {
Y
Yu Yang 已提交
791
             self = gpu_place;
C
chengduoZH 已提交
792 793
           })
      .def("set_place", [](platform::Place &self,
C
chengduoZH 已提交
794 795
                           const platform::CUDAPinnedPlace &cuda_pinned_place) {
        self = cuda_pinned_place;
C
chengduoZH 已提交
796
      });
Y
Yu Yang 已提交
797

Y
Yu Yang 已提交
798 799 800
  py::class_<OperatorBase>(m, "Operator")
      .def_static("create",
                  [](py::bytes protobin) {
801
                    proto::OpDesc desc;
Y
Yu Yang 已提交
802 803 804 805 806
                    PADDLE_ENFORCE(desc.ParsePartialFromString(protobin),
                                   "Cannot parse user input to OpDesc");
                    PADDLE_ENFORCE(desc.IsInitialized(),
                                   "User OpDesc is not initialized, reason %s",
                                   desc.InitializationErrorString());
807
                    return OpRegistry::CreateOp(desc);
Y
Yu Yang 已提交
808
                  })
809
      .def("run",
810
           [](OperatorBase &self, const Scope &scope,
D
dzhwinter 已提交
811 812 813
              const platform::CPUPlace &place) { self.Run(scope, place); })
      .def("run",
           [](OperatorBase &self, const Scope &scope,
D
dzhwinter 已提交
814
              const platform::CUDAPlace &place) { self.Run(scope, place); })
C
chengduoZH 已提交
815 816 817 818 819
      .def("run",
           [](OperatorBase &self, const Scope &scope,
              const platform::CUDAPinnedPlace &place) {
             self.Run(scope, place);
           })
Y
Yu Yang 已提交
820 821 822 823 824 825 826
      .def("type",
           [](const OperatorBase &op) -> std::string { return op.Type(); })
      .def("outputs",
           [](const OperatorBase &op)
               -> std::map<std::string, std::vector<std::string>> {
                 return op.Outputs();
               })
Q
qijun 已提交
827 828
      .def("output_vars",
           [](const OperatorBase &op) { return op.OutputVars(true); })
Y
Yu Yang 已提交
829
      .def("inputs", [](const OperatorBase &op) { return op.Inputs(); })
Q
qijun 已提交
830
      .def("input_vars", [](const OperatorBase &op) { return op.InputVars(); })
Y
Yu Yang 已提交
831 832 833 834
      .def("__str__", &OperatorBase::DebugString)
      .def("no_intermediate_outputs",
           [](const OperatorBase &op) { return op.OutputVars(false); })
      .def("support_gpu", &OperatorBase::SupportGPU);
Y
Yu Yang 已提交
835

F
fengjiayi 已提交
836
  py::class_<framework::Executor>(m, "Executor")
D
dzhwinter 已提交
837
      .def(py::init<const platform::Place &>())
Y
Yancey1989 已提交
838
      .def("close", &Executor::Close)
S
sneaxiy 已提交
839 840 841 842 843
      .def("run", [](Executor &self, const ProgramDesc &prog, Scope *scope,
                     int block_id, bool create_local_scope, bool create_vars) {
        pybind11::gil_scoped_release release;
        self.Run(prog, scope, block_id, create_local_scope, create_vars);
      });
S
sneaxiy 已提交
844

D
dzhwinter 已提交
845
  m.def("init_gflags", framework::InitGflags);
Y
Yang Yu 已提交
846
  m.def("init_glog", framework::InitGLOG);
X
Xin Pan 已提交
847 848
  m.def("init_devices",
        [](bool init_p2p) { framework::InitDevices(init_p2p); });
849

850
  m.def("is_compiled_with_cuda", IsCompiledWithCUDA);
851
  m.def("is_compiled_with_brpc", IsCompiledWithBrpc);
Y
update  
Yancey1989 已提交
852
  m.def("is_compiled_with_dist", IsCompiledWithDIST);
853 854 855 856 857 858
#ifdef PADDLE_WITH_CUDA
  m.def("is_float16_supported", [](const platform::CUDAPlace &place) -> bool {
    // Only GPUs with Compute Capability >= 53 support float16
    return platform::GetCUDAComputeCapability(place.device) >= 53;
  });
#endif
859

860
  m.def("set_feed_variable", framework::SetFeedVariable);
Q
qijun 已提交
861
  m.def("get_fetch_variable", framework::GetFetchVariable);
862
  m.def("get_variable_tensor", framework::GetVariableTensor);
Q
qijun 已提交
863

X
Xin Pan 已提交
864 865
  m.def("_is_program_version_supported", IsProgramVersionSupported);

866 867 868 869 870
  BindProgramDesc(&m);
  BindBlockDesc(&m);
  BindVarDsec(&m);
  BindOpDesc(&m);
  BindConstValue(&m);
Y
Yu Yang 已提交
871

Y
Yu Yang 已提交
872 873 874 875 876 877 878 879 880
  py::class_<framework::LoDRankTable>(m, "LodRankTable")
      .def("items", [](framework::LoDRankTable &table) {
        std::vector<std::pair<size_t, size_t>> res;
        for (auto &item : table.items()) {
          res.push_back({item.index, item.length});
        }
        return res;
      });

Y
Yu Yang 已提交
881
  py::class_<LoDTensorArray>(m, "LoDTensorArray")
S
sneaxiy 已提交
882 883
      .def("__init__",
           [](LoDTensorArray &instance) { new (&instance) LoDTensorArray(); })
Y
Yu Yang 已提交
884 885 886 887 888 889 890 891 892 893
      .def("__getitem__",
           [](LoDTensorArray &self, size_t i) { return &self.at(i); },
           py::return_value_policy::reference)
      .def("__len__", [](LoDTensorArray &self) { return self.size(); })
      .def("__setitem__",
           [](LoDTensorArray &self, size_t i, const LoDTensor &t) {
             PADDLE_ENFORCE_LT(i, self.size());
             self[i].ShareDataWith(t);
             self[i].set_lod(t.lod());
           })
S
sneaxiy 已提交
894 895 896 897 898 899 900
      .def("append",
           [](LoDTensorArray &self, const LoDTensor &t) {
             self.emplace_back();
             self.back().ShareDataWith(t);
             self.back().set_lod(t.lod());
           },
           py::arg("tensor"), "Append a LoDensor to LoDTensorArray.");
Y
Yu Yang 已提交
901

D
dzhwinter 已提交
902 903 904
  m.def("IsInplace",
        [](std::string op) -> bool { return operators::IsInplace(op); });

Y
Yu Yang 已提交
905
  m.def("op_support_gpu", OpSupportGPU);
D
Dong Zhihong 已提交
906
#ifdef PADDLE_WITH_CUDA
D
Dong Zhihong 已提交
907
  m.def("get_cuda_device_count", platform::GetCUDADeviceCount);
D
dangqingqing 已提交
908

P
peizhilin 已提交
909
#ifndef _WIN32
D
dangqingqing 已提交
910 911 912
  m.def("nvprof_init", platform::CudaProfilerInit);
  m.def("nvprof_start", platform::CudaProfilerStart);
  m.def("nvprof_stop", platform::CudaProfilerStop);
D
Dong Zhihong 已提交
913
#endif
P
peizhilin 已提交
914
#endif
Y
Yu Yang 已提交
915

916 917 918 919
  py::enum_<platform::ProfilerState>(m, "ProfilerState", py::arithmetic())
      .value("kDisabled", platform::ProfilerState::kDisabled)
      .value("kCPU", platform::ProfilerState::kCPU)
      .value("kCUDA", platform::ProfilerState::kCUDA)
920
      .value("kAll", platform::ProfilerState::kAll)
921 922 923 924 925 926 927 928 929 930 931 932 933
      .export_values();

  py::enum_<platform::EventSortingKey>(m, "EventSortingKey", py::arithmetic())
      .value("kDefault", platform::EventSortingKey::kDefault)
      .value("kCalls", platform::EventSortingKey::kCalls)
      .value("kTotal", platform::EventSortingKey::kTotal)
      .value("kMin", platform::EventSortingKey::kMin)
      .value("kMax", platform::EventSortingKey::kMax)
      .value("kAve", platform::EventSortingKey::kAve)
      .export_values();

  m.def("enable_profiler", platform::EnableProfiler);
  m.def("disable_profiler", platform::DisableProfiler);
X
Xin Pan 已提交
934
  m.def("is_profiler_enabled", platform::IsProfileEnabled);
935
  m.def("reset_profiler", platform::ResetProfiler);
936
  m.def("get_pass", [](const std::string &pass_type) {
W
WangZhen 已提交
937 938 939
    auto pass = framework::ir::PassRegistry::Instance().Get(pass_type);
    return std::shared_ptr<framework::ir::Pass>(std::move(pass));
  });
Y
Yu Yang 已提交
940

941 942
  py::class_<ir::Pass, std::shared_ptr<ir::Pass>> pass(m, "Pass");
  pass.def(py::init())
W
WangZhen 已提交
943
      .def("has", &ir::Pass::Has)
944 945 946
      .def("set_not_owned",
           [](ir::Pass &self, const std::string &attr_name, ProgramDesc &attr) {
             self.SetNotOwned<ProgramDesc>(attr_name, &attr);
W
WangZhen 已提交
947
           })
948
      .def(
949
          "set",
950 951 952
          [](ir::Pass &self, const std::string &name, const std::string &attr) {
            self.Set<std::string>(name, new std::string(attr));
          })
953 954
      .def("set", [](ir::Pass &self, const std::string &name,
                     int val) { self.Set<const int>(name, new int(val)); })
F
flame 已提交
955 956 957 958
      .def("type", &ir::Pass::Type)
      .def("apply", [](ir::Pass &self, std::shared_ptr<ir::Graph> graph) {
        std::unique_ptr<ir::Graph> origin_graph(graph.get());
        auto optim_graph = self.Apply(std::move(origin_graph));
W
WangZhen 已提交
959
        optim_graph.release();
F
flame 已提交
960
      });
961

X
fix  
Xin Pan 已提交
962 963
  py::class_<ir::PassBuilder, std::shared_ptr<ir::PassBuilder>> pb(
      m, "PassBuilder");
964 965 966 967 968 969 970 971 972 973 974 975 976 977
  pb.def(py::init())
      .def("append_pass",
           [](ir::PassBuilder &self,
              const std::string &pass_type) -> std::shared_ptr<ir::Pass> {
             return self.AppendPass(pass_type);
           })
      .def("all_passes", [](ir::PassBuilder &self) { return self.AllPasses(); })
      .def("insert_pass",
           [](ir::PassBuilder &self, size_t idx, const std::string &pass_type) {
             return self.InsertPass(idx, pass_type);
           })
      .def("remove_pass",
           [](ir::PassBuilder &self, size_t idx) { self.RemovePass(idx); });

Y
yuyang18 已提交
978
  // -- python binds for parallel executor.
X
Xin Pan 已提交
979

Y
yuyang18 已提交
980
  py::class_<ParallelExecutor> pe(m, "ParallelExecutor");
C
chengduo 已提交
981 982 983 984
  py::class_<ExecutionStrategy> exec_strategy(pe, "ExecutionStrategy", R"DOC(
    ExecutionStrategy allows the user to more preciously control how to run
    the program in ParallelExecutor by setting the property.

C
chengduo 已提交
985 986 987 988 989 990 991 992 993 994 995
    Examples:
        .. code-block:: python

          exec_strategy = fluid.ExecutionStrategy()
          exec_strategy.num_threads = 4

          train_exe = fluid.ParallelExecutor(use_cuda=True,
                                             loss_name=loss.name,
                                             exec_strategy=exec_strategy)

          train_loss, = train_exe.run([loss.name], feed=feed_dict)
C
chengduo 已提交
996 997 998

        )DOC");

Y
yuyang18 已提交
999
  exec_strategy.def(py::init())
Y
yuyang18 已提交
1000 1001 1002 1003 1004
      .def_property(
          "num_threads",
          [](const ExecutionStrategy &self) { return self.num_threads_; },
          [](ExecutionStrategy &self, size_t num_threads) {
            self.num_threads_ = num_threads;
C
chengduo 已提交
1005 1006 1007 1008 1009 1010 1011 1012 1013 1014
          },
          R"DOC(The type is INT, num_threads represents the size of thread pool that
            used to run the operators of the current program in ParallelExecutor.
            If :math:`num\_threads=1`, all the operators will execute one by one,
            but the order maybe difference between iterations.
            If it is not set, it will be set in ParallelExecutor according to the
            device type and device count, for GPU, :math:`num\_threads=device\_count*4`, for CPU,
            :math:`num\_threads=CPU\_NUM*4`, the explanation of:math:`CPU\_NUM` is in ParallelExecutor.
            if it is not set, ParallelExecutor will get the cpu count by calling
            `multiprocessing.cpu_count()`. Default 0.)DOC")
Y
yuyang18 已提交
1015
      .def_property(
1016 1017 1018 1019
          "use_cuda",
          [](const ExecutionStrategy &self) { return self.use_cuda_; },
          [](ExecutionStrategy &self, bool use_cuda) {
            self.use_cuda_ = use_cuda;
C
chengduo 已提交
1020 1021 1022 1023
          })  // FIXME(chengduo): Doesn't add doc for 'use_cuda', use_cuda may
      // make user confuse, because ParallelExecutor has a parameter named
      // 'use_cuda' too, in current implementation, ParallelExecutor's
      // 'use_cuda' will rewrite ExecutionStrategy's 'use_cuda'.
Y
yuyang18 已提交
1024 1025 1026 1027 1028
      .def_property(
          "allow_op_delay",
          [](const ExecutionStrategy &self) { return self.allow_op_delay_; },
          [](ExecutionStrategy &self, bool allow_op_delay) {
            self.allow_op_delay_ = allow_op_delay;
C
chengduo 已提交
1029 1030 1031 1032
          },
          R"DOC(The type is BOOL, allow_op_delay represents whether to delay the
                communication operators to run, it may make the execution faster.
                Note that in some models, allow_op_delay may cause program hang. Default False.)DOC")
Y
yuyang18 已提交
1033 1034 1035 1036 1037 1038 1039
      .def_property(
          "num_iteration_per_drop_scope",
          [](const ExecutionStrategy &self) {
            return self.num_iteration_per_drop_scope_;
          },
          [](ExecutionStrategy &self, size_t num_iteration_per_drop_scope) {
            self.num_iteration_per_drop_scope_ = num_iteration_per_drop_scope;
C
chengduo 已提交
1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050
          },
          R"DOC(The type is INT, num_iteration_per_drop_scope indicates how
                many iterations to clean up the temp variables which
                is generated during execution. It may make the execution faster,
                because the temp variable's shape maybe the same between two iterations. Default 100.

                NOTES:
                    1. If you fetch data when calling the 'run', the ParallelExecutor
                       will clean up the temp variables at the end of the current iteration.
                    2. In some NLP model, it may cause the GPU memory is insufficient,
                       in this case, you should reduce `num_iteration_per_drop_scope`.
1051
              )DOC")
Q
Qiao Longfei 已提交
1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062
      .def_property(
          "num_iteration_per_run",
          [](const ExecutionStrategy &self) {
            return self.num_iteration_per_run_;
          },
          [](ExecutionStrategy &self, size_t num_iteration_per_run) {
            self.num_iteration_per_run_ = num_iteration_per_run;
          },
          R"DOC(This config that how many iteration the executor will run when
                user call pe.run() in python
              )DOC")
1063 1064 1065 1066 1067
      .def_property("_dry_run",
                    [](const ExecutionStrategy &self) { return self.dry_run_; },
                    [](ExecutionStrategy &self, bool dry_run) {
                      self.dry_run_ = dry_run;
                    });
C
chengduo 已提交
1068

Y
yuyang18 已提交
1069
  exec_strategy.def_property(
Y
yuyang18 已提交
1070 1071 1072 1073 1074 1075 1076
      "use_experimental_executor",
      [](const ExecutionStrategy &self) {
        return self.type_ == ExecutionStrategy::kExperimental;
      },
      [](ExecutionStrategy &self, bool experimental) {
        self.type_ = experimental ? ExecutionStrategy::kExperimental
                                  : ExecutionStrategy::kDefault;
Y
yuyang18 已提交
1077 1078
      });

C
chengduo 已提交
1079 1080 1081 1082
  py::class_<BuildStrategy> build_strategy(pe, "BuildStrategy", R"DOC(
    BuildStrategy allows the user to more preciously control how to
    build the SSA Graph in ParallelExecutor by setting the property.

C
chengduo 已提交
1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093
    Examples:
        .. code-block:: python

          build_strategy = fluid.BuildStrategy()
          build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce

          train_exe = fluid.ParallelExecutor(use_cuda=True,
                                             loss_name=loss.name,
                                             build_strategy=build_strategy)

          train_loss, = train_exe.run([loss.name], feed=feed_dict)
C
chengduo 已提交
1094
)DOC");
Y
yuyang18 已提交
1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110

  py::enum_<BuildStrategy::ReduceStrategy>(build_strategy, "ReduceStrategy")
      .value("Reduce", BuildStrategy::ReduceStrategy::kReduce)
      .value("AllReduce", BuildStrategy::ReduceStrategy::kAllReduce);
  py::enum_<BuildStrategy::GradientScaleStrategy>(build_strategy,
                                                  "GradientScaleStrategy")
      .value("CoeffNumDevice",
             BuildStrategy::GradientScaleStrategy::kCoeffNumDevice)
      .value("One", BuildStrategy::GradientScaleStrategy::kOne)
      .value("Customized", BuildStrategy::GradientScaleStrategy::kCustomized);

  build_strategy.def(py::init())
      .def_property(
          "reduce_strategy",
          [](const BuildStrategy &self) { return self.reduce_; },
          [](BuildStrategy &self, BuildStrategy::ReduceStrategy strategy) {
X
Xin Pan 已提交
1111
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
Y
yuyang18 已提交
1112
            self.reduce_ = strategy;
C
chengduo 已提交
1113 1114 1115 1116 1117 1118 1119
          },
          R"DOC(The type is STR, there are two reduce strategies in ParallelExecutor,
                  'AllReduce' and 'Reduce'. If you want that all the parameters'
                  optimization are done on all devices independently, you should choose 'AllReduce';
                  if you choose 'Reduce', all the parameters' optimization will be evenly distributed
                  to different devices, and then broadcast the optimized parameter to other devices.
                  In some models, `Reduce` is faster. Default 'AllReduce'. )DOC")
Y
yuyang18 已提交
1120 1121 1122 1123 1124
      .def_property(
          "gradient_scale_strategy",
          [](const BuildStrategy &self) { return self.gradient_scale_; },
          [](BuildStrategy &self,
             BuildStrategy::GradientScaleStrategy strategy) {
X
Xin Pan 已提交
1125
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
Y
yuyang18 已提交
1126
            self.gradient_scale_ = strategy;
C
chengduo 已提交
1127 1128 1129 1130 1131 1132
          },
          R"DOC(The type is STR, there are three ways of defining :math:`loss@grad` in
                   ParallelExecutor, 'CoeffNumDevice', 'One' and 'Customized'. By default,
                   ParallelExecutor sets the :math:`loss@grad` according to the number of devices.
                   If you want to customize :math:`loss@grad`, you can choose 'Customized'.
                   Default 'CoeffNumDevice'.)DOC")
Y
yuyang18 已提交
1133 1134 1135 1136
      .def_property(
          "debug_graphviz_path",
          [](const BuildStrategy &self) { return self.debug_graphviz_path_; },
          [](BuildStrategy &self, const std::string &path) {
X
Xin Pan 已提交
1137
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
Y
yuyang18 已提交
1138
            self.debug_graphviz_path_ = path;
C
chengduo 已提交
1139 1140 1141 1142
          },
          R"DOC(The type is STR, debug_graphviz_path indicate the path that
                    writing the SSA Graph to file in the form of graphviz, you.
                    It is useful for debugging. Default "")DOC")
S
sneaxiy 已提交
1143 1144 1145 1146 1147 1148
      .def_property(
          "enable_sequential_execution",
          [](const BuildStrategy &self) {
            return self.enable_sequential_execution_;
          },
          [](BuildStrategy &self, bool b) {
X
Xin Pan 已提交
1149
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
S
sneaxiy 已提交
1150 1151 1152 1153 1154 1155 1156 1157 1158
            self.enable_sequential_execution_ = b;
          },
          R"DOC(The type is BOOL. If set True, the execution order of ops would be the same as what is in the program. Default False.)DOC")
      .def_property(
          "remove_unnecessary_lock",
          [](const BuildStrategy &self) {
            return self.remove_unnecessary_lock_;
          },
          [](BuildStrategy &self, bool b) {
X
Xin Pan 已提交
1159
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
S
sneaxiy 已提交
1160 1161
            self.remove_unnecessary_lock_ = b;
          },
S
sneaxiy 已提交
1162
          R"DOC(The type is BOOL. If set True, some locks in GPU ops would be released and ParallelExecutor would run faster. Default True.)DOC")
1163 1164 1165 1166 1167 1168
      .def_property(
          "num_trainers",
          [](const BuildStrategy &self) { return self.num_trainers_; },
          [](BuildStrategy &self, int num_trainers) {
            self.num_trainers_ = num_trainers;
          })
1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180
      .def_property(
          "trainers_endpoints",
          [](const BuildStrategy &self) { return self.trainers_endpoints_; },
          [](BuildStrategy &self,
             const std::vector<std::string> &trainers_endpoints) {
            self.trainers_endpoints_ = trainers_endpoints;
          })
      .def_property("trainer_id",
                    [](const BuildStrategy &self) { return self.trainer_id_; },
                    [](BuildStrategy &self, int trainer_id) {
                      self.trainer_id_ = trainer_id;
                    })
C
chengduo 已提交
1181 1182 1183 1184 1185 1186
      .def_property(
          "fuse_elewise_add_act_ops",
          [](const BuildStrategy &self) {
            return self.fuse_elewise_add_act_ops_;
          },
          [](BuildStrategy &self, bool b) {
X
Xin Pan 已提交
1187
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
C
chengduo 已提交
1188 1189 1190 1191 1192
            self.fuse_elewise_add_act_ops_ = b;
          },
          R"DOC(The type is BOOL, fuse_elewise_add_act_ops indicate whether
                     to fuse elementwise_add_op and activation_op,
                     it may make the execution faster. Default False)DOC")
1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206
      .def_property(
          "fuse_relu_depthwise_conv",
          [](const BuildStrategy &self) {
            return self.fuse_relu_depthwise_conv_;
          },
          [](BuildStrategy &self, bool b) {
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
            self.fuse_relu_depthwise_conv_ = b;
          },
          R"DOC(The type is BOOL, fuse_relu_depthwise_conv indicate whether
                      to fuse relu and depthwise_conv2d,
                      it will save GPU memory and may make the execution faster.
                      This options is only available in GPU devices.
                      Default False)DOC")
D
dzhwinter 已提交
1207 1208 1209 1210
      .def_property(
          "memory_optimize",
          [](const BuildStrategy &self) { return self.memory_optimize_; },
          [](BuildStrategy &self, bool b) { self.memory_optimize_ = b; })
1211 1212 1213 1214
      .def_property(
          "is_distribution",
          [](const BuildStrategy &self) { return self.is_distribution_; },
          [](BuildStrategy &self, bool b) { self.is_distribution_ = b; })
Q
can run  
Qiao Longfei 已提交
1215 1216 1217
      .def_property("async_mode",
                    [](const BuildStrategy &self) { return self.async_mode_; },
                    [](BuildStrategy &self, bool b) { self.async_mode_ = b; })
D
dzhwinter 已提交
1218 1219 1220 1221
      .def_property(
          "enable_inplace",
          [](const BuildStrategy &self) { return self.enable_inplace_; },
          [](BuildStrategy &self, bool b) { self.enable_inplace_ = b; })
1222
      .def("_finalize_strategy_and_create_passes",
X
fix  
Xin Pan 已提交
1223
           [](BuildStrategy &self) -> std::shared_ptr<ir::PassBuilder> {
1224 1225 1226 1227 1228
             return self.CreatePassesFromStrategy(true);
           },
           R"DOC(Allow user to customized passes. Normally model-specific
                optimization passes should be defined in this way. BuildStrategy
                cannot be updated after being finalized.)DOC");
Y
yuyang18 已提交
1229 1230

  pe.def(py::init<const std::vector<platform::Place> &,
X
Xin Pan 已提交
1231
                  const std::unordered_set<std::string> &, const std::string &,
X
Xin Pan 已提交
1232
                  Scope *, std::vector<Scope *> &, const ExecutionStrategy &,
Q
Qiao Longfei 已提交
1233
                  const BuildStrategy &, std::vector<ir::Graph *>>())
Y
Yu Yang 已提交
1234 1235 1236 1237
      // NOTE: even we return a vec<Scope*>* to Python use reference policy.
      // We still cannot get local_scope from this vector, since the element
      // of vec<Scope*> will be freed by Python GC. We can only return Scope*
      // one by one and mark them as reference.
1238 1239 1240 1241 1242
      .def("local_scopes",
           [](ParallelExecutor &self) -> std::vector<Scope *> * {
             return &self.GetLocalScopes();
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
1243 1244 1245 1246
      .def("feed_tensors_into_local_scopes",
           &ParallelExecutor::FeedTensorsIntoLocalScopes)
      .def("feed_and_split_tensor_into_local_scopes",
           &ParallelExecutor::FeedAndSplitTensorIntoLocalScopes)
S
sneaxiy 已提交
1247 1248 1249 1250 1251 1252
      .def("run", [](ParallelExecutor &self,
                     const std::vector<std::string> &fetch_tensors,
                     const std::string &fetched_var_name) {
        pybind11::gil_scoped_release release;
        self.Run(fetch_tensors, fetched_var_name);
      });
Y
Yu Yang 已提交
1253

1254
  BindRecordIOWriter(&m);
W
Wang Guibao 已提交
1255
  BindAsyncExecutor(&m);
F
flame 已提交
1256 1257
  BindGraph(&m);
  BindNode(&m);
F
flame 已提交
1258
  BindInferenceApi(&m);
L
Luo Tao 已提交
1259
}
1260
}  // namespace pybind
1261
}  // namespace paddle