pybind.cc 64.0 KB
Newer Older
1
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
2 3 4 5 6

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

7
http://www.apache.org/licenses/LICENSE-2.0
8 9 10 11 12 13

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
L
lgone2000 已提交
14
#include <Python.h>
C
chengduoZH 已提交
15 16
#include <algorithm>
#include <map>
S
sneaxiy 已提交
17
#include <memory>
C
chengduoZH 已提交
18 19 20
#include <mutex>  // NOLINT // for call_once
#include <string>
#include <unordered_map>
21
#include <unordered_set>
C
chengduoZH 已提交
22 23
#include <utility>
#include <vector>
24

Y
Yi Wang 已提交
25 26 27
#include "paddle/fluid/framework/executor.h"
#include "paddle/fluid/framework/feed_fetch_method.h"
#include "paddle/fluid/framework/framework.pb.h"
S
sneaxiy 已提交
28
#include "paddle/fluid/framework/garbage_collector.h"
29
#include "paddle/fluid/framework/ir/alloc_continuous_space_for_grad_pass.h"
30
#include "paddle/fluid/framework/ir/pass_builder.h"
Y
Yi Wang 已提交
31 32 33
#include "paddle/fluid/framework/lod_rank_table.h"
#include "paddle/fluid/framework/lod_tensor.h"
#include "paddle/fluid/framework/lod_tensor_array.h"
S
sneaxiy 已提交
34
#include "paddle/fluid/framework/op_info.h"
35
#include "paddle/fluid/framework/op_registry.h"
Y
Yu Yang 已提交
36
#include "paddle/fluid/framework/parallel_executor.h"
Y
Yi Wang 已提交
37
#include "paddle/fluid/framework/prune.h"
Y
Refine  
Yu Yang 已提交
38
#include "paddle/fluid/framework/reader.h"
S
sneaxiy 已提交
39
#include "paddle/fluid/framework/scope_pool.h"
Y
Yi Wang 已提交
40
#include "paddle/fluid/framework/selected_rows.h"
X
Xin Pan 已提交
41
#include "paddle/fluid/framework/version.h"
Y
Refine  
Yu Yang 已提交
42
#include "paddle/fluid/memory/allocation/allocator_strategy.h"
43
#include "paddle/fluid/memory/allocation/legacy_allocator.h"
D
dzhwinter 已提交
44
#include "paddle/fluid/operators/activation_op.h"
S
sneaxiy 已提交
45
#include "paddle/fluid/operators/py_func_op.h"
S
sneaxiy 已提交
46
#include "paddle/fluid/operators/reader/lod_tensor_blocking_queue.h"
47
#include "paddle/fluid/platform/cpu_helper.h"
Y
Yu Yang 已提交
48
#include "paddle/fluid/platform/cpu_info.h"
Y
Yi Wang 已提交
49
#include "paddle/fluid/platform/enforce.h"
50
#include "paddle/fluid/platform/init.h"
Y
Yi Wang 已提交
51 52
#include "paddle/fluid/platform/place.h"
#include "paddle/fluid/platform/profiler.h"
W
Wang Guibao 已提交
53
#include "paddle/fluid/pybind/async_executor_py.h"
Y
Yi Wang 已提交
54
#include "paddle/fluid/pybind/const_value.h"
D
dongdaxiang 已提交
55
#include "paddle/fluid/pybind/data_set_py.h"
Y
Yi Wang 已提交
56
#include "paddle/fluid/pybind/exception.h"
D
dongdaxiang 已提交
57
#include "paddle/fluid/pybind/fleet_wrapper_py.h"
58
#include "paddle/fluid/pybind/imperative.h"
F
flame 已提交
59
#include "paddle/fluid/pybind/inference_api.h"
F
flame 已提交
60
#include "paddle/fluid/pybind/ir.h"
61

W
wopeizl 已提交
62
#ifndef _WIN32
D
dongdaxiang 已提交
63
#include "paddle/fluid/pybind/nccl_wrapper_py.h"
W
wopeizl 已提交
64
#endif
65 66
#include "paddle/fluid/pybind/protobuf.h"
#include "paddle/fluid/pybind/pybind.h"  // NOLINT
S
sneaxiy 已提交
67
#include "paddle/fluid/pybind/reader_py.h"
Y
Yu Yang 已提交
68
#include "paddle/fluid/pybind/recordio.h"
Y
Yi Wang 已提交
69
#include "paddle/fluid/pybind/tensor_py.h"
70
#include "paddle/fluid/string/to_string.h"
71

D
Dong Zhihong 已提交
72
#ifdef PADDLE_WITH_CUDA
P
peizhilin 已提交
73
#ifndef _WIN32
Y
Yi Wang 已提交
74
#include "paddle/fluid/operators/nccl/nccl_gpu_common.h"
P
peizhilin 已提交
75
#endif
Y
Yi Wang 已提交
76 77
#include "paddle/fluid/platform/cuda_profiler.h"
#include "paddle/fluid/platform/gpu_info.h"
D
Dong Zhihong 已提交
78 79
#endif

80 81 82 83
#ifdef PADDLE_WITH_DISTRIBUTE
#include "paddle/fluid/pybind/communicator_py.h"
#endif

M
minqiyang 已提交
84 85
#include "pybind11/stl.h"

86 87 88 89
DEFINE_bool(reader_queue_speed_test_mode, false,
            "If set true, the queue.pop will only get data from queue but not "
            "remove the data from queue for speed testing");

Q
Qiao Longfei 已提交
90 91 92
// disable auto conversion to list in Python
PYBIND11_MAKE_OPAQUE(paddle::framework::LoDTensorArray);

93
namespace paddle {
94
namespace pybind {
95
bool IsCompiledWithCUDA() {
96
#ifndef PADDLE_WITH_CUDA
Q
qijun 已提交
97 98 99 100 101 102
  return false;
#else
  return true;
#endif
}

103 104 105 106 107 108 109 110
bool IsCompiledWithMKLDNN() {
#ifndef PADDLE_WITH_MKLDNN
  return false;
#else
  return true;
#endif
}

111 112 113 114 115 116 117 118
bool IsCompiledWithNGRAPH() {
#ifndef PADDLE_WITH_NGRAPH
  return false;
#else
  return true;
#endif
}

119
bool IsCompiledWithBrpc() {
120
#ifndef PADDLE_WITH_DISTRIBUTE
121 122
  return false;
#endif
123 124 125 126 127 128

#ifdef PADDLE_WITH_GRPC
  return false;
#endif

  return true;
129 130
}

Y
update  
Yancey1989 已提交
131
bool IsCompiledWithDIST() {
Y
Yancey1989 已提交
132
#ifdef PADDLE_WITH_DISTRIBUTE
Y
update  
Yancey1989 已提交
133 134 135 136 137 138
  return true;
#else
  return false;
#endif
}

S
sneaxiy 已提交
139 140 141 142 143 144 145 146 147 148
template <typename PlaceType1, typename PlaceType2>
static inline bool IsSamePlace(const PlaceType1 &p1, const PlaceType2 &p2) {
  return paddle::platform::Place(p1) == paddle::platform::Place(p2);
}

template <typename PlaceType>
static inline int PlaceIndex(const PlaceType &p) {
  return static_cast<int>(paddle::platform::Place(p).which());
}

149 150 151 152 153 154
#ifdef PADDLE_WITH_AVX
PYBIND11_MODULE(core_avx, m) {
#else
PYBIND11_MODULE(core_noavx, m) {
#endif

Y
Yu Yang 已提交
155 156 157
  // Not used, just make sure cpu_info.cc is linked.
  paddle::platform::CpuTotalPhysicalMemory();

Y
Refine  
Yu Yang 已提交
158
  paddle::memory::allocation::UseAllocatorStrategyGFlag();
S
sneaxiy 已提交
159

160
  m.doc() = "C++ core of PaddlePaddle";
161

162 163 164 165
  // using framework in this function. Since it is inside a function, it will
  // not cause namespace pollution.
  using namespace paddle::framework;  // NOLINT

166
  BindException(&m);
Y
Yu Yang 已提交
167

168 169
  m.def("set_num_threads", &platform::SetNumThreads);

S
sneaxiy 已提交
170
  m.def(
S
sneaxiy 已提交
171
      "_append_python_callable_object_and_return_id",
S
sneaxiy 已提交
172 173 174 175
      [](py::object py_obj) -> size_t {
        return paddle::operators::AppendPythonCallableObjectAndReturnId(py_obj);
      });

S
sneaxiy 已提交
176 177 178
  m.def("_get_use_default_grad_op_desc_maker_ops",
        [] { return OpInfoMap::Instance().GetUseDefaultGradOpDescMakerOps(); });

S
sneaxiy 已提交
179 180 181
  // NOTE(zjl): ctest would load environment variables at the beginning even
  // though we have not `import paddle.fluid as fluid`. So we add this API
  // to enable eager deletion mode in unittest.
S
sneaxiy 已提交
182
  m.def("_set_eager_deletion_mode", &paddle::framework::SetEagerDeletionMode);
S
sneaxiy 已提交
183

184
  m.def("_set_fuse_parameter_group_size",
185
        &paddle::framework::ir::SetFuseParameterGroupsSize);
186
  m.def("_set_fuse_parameter_memory_size",
187
        &paddle::framework::ir::SetFuseParameterMemorySize);
188

S
sneaxiy 已提交
189 190 191
  m.add_object("_cleanup",
               py::capsule([]() { ScopePool::Instance().Clear(); }));

192 193 194 195 196 197 198
  m.def("get_mem_usage", [](int device) {
    return memory::allocation::GPUMemMonitor.GetMemUsage(device);
  });

  m.def("print_mem_usage",
        []() { return memory::allocation::GPUMemMonitor.PrintMemUsage(); });

199
  BindImperative(&m);
200

201
  py::class_<Tensor>(m, "Tensor", py::buffer_protocol())
202
      .def("__array__", [](Tensor &self) { return TensorToPyArray(self); })
S
sneaxiy 已提交
203 204
      .def("_is_initialized",
           [](const Tensor &self) { return self.IsInitialized(); })
Y
yuyang18 已提交
205
      .def("_get_dims",
206
           [](const Tensor &self) { return vectorize(self.dims()); })
Y
yuyang18 已提交
207
      .def("_set_dims",
Q
qijun 已提交
208
           [](Tensor &self, const std::vector<int64_t> &dim) {
Y
Yu Yang 已提交
209
             self.Resize(make_ddim(dim));
Y
Yu Yang 已提交
210
           })
Y
yuyang18 已提交
211
      .def("_set_layout",
D
dzhwinter 已提交
212 213 214
           [](Tensor &self, const std::string &layout) {
             self.set_layout(StringToDataLayout(layout));
           })
Y
yuyang18 已提交
215
      .def("_alloc_float",
D
dzhwinter 已提交
216
           [](Tensor &self, paddle::platform::CUDAPlace &place) {
Q
qijun 已提交
217
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
218
           })
Y
yuyang18 已提交
219
      .def("_alloc_float",
Y
Yu Yang 已提交
220
           [](Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
221
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
222
           })
Y
yuyang18 已提交
223
      .def("_alloc_int",
Y
Yu Yang 已提交
224
           [](Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
225
             self.mutable_data<int>(place);
Y
Yu Yang 已提交
226
           })
Y
yuyang18 已提交
227
      .def("_alloc_int",
D
dzhwinter 已提交
228
           [](Tensor &self, paddle::platform::CUDAPlace &place) {
Q
qijun 已提交
229
             self.mutable_data<int>(place);
Q
qijun 已提交
230
           })
Y
yuyang18 已提交
231
      .def("_alloc_int",
C
chengduoZH 已提交
232 233 234
           [](Tensor &self, paddle::platform::CUDAPinnedPlace &place) {
             self.mutable_data<int>(place);
           })
Y
yuyang18 已提交
235
      .def("_alloc_float",
C
chengduoZH 已提交
236 237 238
           [](Tensor &self, paddle::platform::CUDAPinnedPlace &place) {
             self.mutable_data<float>(place);
           })
Z
Zeng Jinle 已提交
239
      .def("_clear", &Tensor::clear)
Y
Yu Yang 已提交
240 241
      .def("set", PyCPUTensorSetFromArray<float>)
      .def("set", PyCPUTensorSetFromArray<int>)
242
      .def("set", PyCPUTensorSetFromArray<double>)
243
      .def("set", PyCPUTensorSetFromArray<int64_t>)
Y
Yu Yang 已提交
244
      .def("set", PyCPUTensorSetFromArray<bool>)
245
      .def("set", PyCPUTensorSetFromArray<uint16_t>)
F
fengjiayi 已提交
246
      .def("set", PyCPUTensorSetFromArray<uint8_t>)
Q
qingqing01 已提交
247
      .def("set", PyCPUTensorSetFromArray<int8_t>)
248
#ifdef PADDLE_WITH_CUDA
Y
Yu Yang 已提交
249 250
      .def("set", PyCUDATensorSetFromArray<float>)
      .def("set", PyCUDATensorSetFromArray<int>)
251
      .def("set", PyCUDATensorSetFromArray<double>)
252
      .def("set", PyCUDATensorSetFromArray<int64_t>)
Y
Yu Yang 已提交
253
      .def("set", PyCUDATensorSetFromArray<bool>)
254
      .def("set", PyCUDATensorSetFromArray<uint16_t>)
F
fengjiayi 已提交
255
      .def("set", PyCUDATensorSetFromArray<uint8_t>)
Q
qingqing01 已提交
256
      .def("set", PyCUDATensorSetFromArray<int8_t>)
C
chengduoZH 已提交
257 258 259 260 261 262
      .def("set", PyCUDAPinnedTensorSetFromArray<float>)
      .def("set", PyCUDAPinnedTensorSetFromArray<int>)
      .def("set", PyCUDAPinnedTensorSetFromArray<double>)
      .def("set", PyCUDAPinnedTensorSetFromArray<int64_t>)
      .def("set", PyCUDAPinnedTensorSetFromArray<bool>)
      .def("set", PyCUDAPinnedTensorSetFromArray<uint16_t>)
F
fengjiayi 已提交
263
      .def("set", PyCUDAPinnedTensorSetFromArray<uint8_t>)
Q
qingqing01 已提交
264
      .def("set", PyCUDAPinnedTensorSetFromArray<int8_t>)
Q
qijun 已提交
265
#endif
266
      .def("shape", [](Tensor &self) { return vectorize(self.dims()); })
Y
yuyang18 已提交
267 268 269 270
      .def("_set_float_element", TensorSetElement<float>)
      .def("_get_float_element", TensorGetElement<float>)
      .def("_set_double_element", TensorSetElement<double>)
      .def("_get_double_element", TensorGetElement<double>)
X
xuezhong 已提交
271
      .def("_place", [](Tensor &self) { return self.place(); })
W
wopeizl 已提交
272
      .def("_dtype", [](Tensor &self) { return self.type(); })
273 274 275 276 277 278
      .def("__getitem__", PySliceTensor, py::return_value_policy::reference)
      .def("__str__", [](const Tensor &self) {
        std::stringstream ostr;
        ostr << self;
        return ostr.str();
      });
Y
Yu Yang 已提交
279

X
Xin Pan 已提交
280 281 282 283 284 285 286 287 288
  py::class_<LoDTensor, Tensor>(m, "LoDTensor", R"DOC(
    LoDTensor is a Tensor with optional LoD information.

    np.array(lod_tensor) can convert LoDTensor to numpy array.
    lod_tensor.lod() can retrieve the LoD information.

    LoD is short for Level of Details and is usually used for varied sequence
    length. You can skip the following comment if you don't need optional LoD.

289 290
    For example, a LoDTensor X can look like the example below. It contains
    2 sequences. The first has length 2 and the second has length 3, as
Z
Zeng Jinle 已提交
291
    described by x.lod.
X
Xin Pan 已提交
292

Z
Zeng Jinle 已提交
293 294 295
    The first tensor dimension 5=2+3 is calculated from LoD if it's available.
    It means the total number of sequence element. In X, each element has 2
    columns, hence [5, 2].
X
Xin Pan 已提交
296

Z
Zeng Jinle 已提交
297
    x.lod  = [[2, 3]]
298

Z
Zeng Jinle 已提交
299
    x.data = [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]]
X
Xin Pan 已提交
300

Z
Zeng Jinle 已提交
301
    x.shape = [5, 2]
X
Xin Pan 已提交
302

Z
Zeng Jinle 已提交
303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319
    LoD can have multiple levels (for example, a paragraph can have multiple
    sentences and a sentence can have multiple words). In the following
    LodTensor Y, the lod_level is 2. It means there are 2 sequence, the
    first sequence length is 2 (has 2 sub-sequences), the second one's
    length is 1. The first sequence's 2 sub-sequences have length 2 and 2,
    respectively. And the second sequence's 1 sub-sequence has length 3.

    y.lod = [[2 1], [2 2 3]]

    y.shape = [2+2+3, ...]

    Examples:
        .. code-block:: python

          import paddle.fluid as fluid

          t = fluid.LoDTensor()
X
Xin Pan 已提交
320 321 322 323 324 325 326 327 328 329 330 331

  Note:
      In above description, LoD is length-based. In Paddle internal
      implementation, lod is offset-based. Hence, internally,
      y.lod is represented as [[0, 2, 3], [0, 2, 4, 7]] (length-based
      equivlent would be [[2-0, 3-2], [2-0, 4-2, 7-4]]).

      Sometimes LoD is called recursive_sequence_length to be more
      self-explanatory. In this case, it must be length-based. Due to history
      reasons. when LoD is called lod in public API, it might be offset-based.
      Users should be careful about it.
        )DOC")
332
      .def("__array__", [](Tensor &self) { return TensorToPyArray(self); })
333 334 335 336 337 338 339 340 341 342 343 344 345 346
      .def("__init__",
           [](LoDTensor &instance, const std::vector<std::vector<size_t>>
                                       &recursive_sequence_lengths) {
             LoD new_lod;
             new_lod.reserve(recursive_sequence_lengths.size());
             std::copy(recursive_sequence_lengths.begin(),
                       recursive_sequence_lengths.end(),
                       std::back_inserter(new_lod));
             LoD new_offset_lod = ConvertToOffsetBasedLoD(new_lod);
             PADDLE_ENFORCE(
                 CheckLoD(new_offset_lod, -1),
                 "the provided recursive_sequence_lengths info is invalid");
             new (&instance) LoDTensor(new_offset_lod);
           })
Y
Yu Yang 已提交
347
      .def("__init__", [](LoDTensor &instance) { new (&instance) LoDTensor(); })
G
gongweibao 已提交
348 349 350 351 352
      // We implement offset based LOD in C++ while we use length based with
      // Python API. So we changed set_lod to set_recursive_sequence_lengths to
      // avoid misuse.
      // The discussion is here:
      // https://github.com/PaddlePaddle/Paddle/issues/10855
D
dangqingqing 已提交
353
      .def("set_lod",
354
           [](LoDTensor &self, const std::vector<std::vector<size_t>> &lod) {
355
             // the input lod is offset-based level-of-detail info
Y
Yu Yang 已提交
356
             LoD new_lod;
357 358
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
359 360
             PADDLE_ENFORCE(CheckLoD(new_lod, vectorize(self.dims()).front()),
                            "the provided lod info is invalid");
361
             self.set_lod(new_lod);
S
sneaxiy 已提交
362 363 364 365 366 367
           },
           py::arg("lod"), R"DOC(
           Set LoD of the LoDTensor.

           Args:
               lod (List[List[int]]): the lod to be set.
Z
Zeng Jinle 已提交
368 369 370 371 372 373 374 375 376 377

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_lod([[0, 2, 5]])
S
sneaxiy 已提交
378
           )DOC")
379 380 381 382 383 384 385 386 387 388 389 390 391 392 393
      .def("set_recursive_sequence_lengths",
           [](LoDTensor &self, const std::vector<std::vector<size_t>>
                                   &recursive_sequence_lengths) {
             // the input recursive_sequence_lengths is length-based
             // level-of-detail info
             LoD new_lod;
             new_lod.reserve(recursive_sequence_lengths.size());
             std::copy(recursive_sequence_lengths.begin(),
                       recursive_sequence_lengths.end(),
                       std::back_inserter(new_lod));
             LoD new_offset_lod = ConvertToOffsetBasedLoD(new_lod);
             PADDLE_ENFORCE(
                 CheckLoD(new_offset_lod, vectorize(self.dims()).front()),
                 "the provided recursive_sequence_lengths info is invalid");
             self.set_lod(new_offset_lod);
S
sneaxiy 已提交
394 395 396 397
           },
           py::arg("recursive_sequence_lengths"), R"DOC(
           Set LoD of the LoDTensor according to recursive sequence length.

S
sneaxiy 已提交
398
           For example, if recursive_sequence_lengths=[[2, 3]], meaning that
399 400
           there are two sequences with length 2 and 3 respectively, the
           corresponding lod would be [[0, 2, 2+3]], i.e, [[0, 2, 5]].
S
sneaxiy 已提交
401 402

           Args:
403
                recursive_sequence_lengths (List[List[int]]): sequence lengths.
Z
Zeng Jinle 已提交
404 405 406 407 408 409 410 411 412 413

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_recursive_sequence_lengths([[2, 3]])
S
sneaxiy 已提交
414
           )DOC")
415 416 417 418 419 420 421 422
      .def("lod",
           [](LoDTensor &self) -> std::vector<std::vector<size_t>> {
             // output the offset-based lod info
             LoD lod = self.lod();
             std::vector<std::vector<size_t>> new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             return new_lod;
S
sneaxiy 已提交
423 424 425 426 427 428
           },
           R"DOC(
           Return the LoD of the LoDTensor.

           Returns:
               out (List[List[int]]): the lod of the LoDTensor.
Z
Zeng Jinle 已提交
429 430 431 432 433 434 435 436 437 438 439

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_lod([[0, 2, 5]])
                 print(t.lod()) # [[0, 2, 5]]
S
sneaxiy 已提交
440
           )DOC")
G
gongweibao 已提交
441
      // Set above comments of set_lod.
442 443 444 445 446 447 448 449
      .def("recursive_sequence_lengths",
           [](LoDTensor &self) -> std::vector<std::vector<size_t>> {
             // output the length-based lod info
             LoD lod = ConvertToLengthBasedLoD(self.lod());
             std::vector<std::vector<size_t>> new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             return new_lod;
S
sneaxiy 已提交
450 451 452 453 454
           },
           R"DOC(
           Return the sequence length of the LoDTensor corresponding to LoD.

           Returns:
455
               out (List[List[int]): the sequence lengths.
Z
Zeng Jinle 已提交
456 457 458 459 460 461 462 463 464 465 466

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_recursive_sequence_lengths([[2, 3]])
                 print(t.recursive_sequence_lengths()) # [[2, 3]]
S
sneaxiy 已提交
467 468 469 470 471 472 473 474 475 476 477 478
           )DOC")
      .def("has_valid_recursive_sequence_lengths",
           [](LoDTensor &self) -> bool {
             // Check that the lod info is valid and match the outermost
             // dimension of the LoDTensor data
             return CheckLoD(self.lod(), vectorize(self.dims()).front());
           },
           R"DOC(
           Check whether the lod of the LoDTensor is valid.

           Returns:
               out (bool): whether the lod is valid.
Z
Zeng Jinle 已提交
479 480 481 482 483 484 485 486 487 488 489

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_recursive_sequence_lengths([[2, 3]])
                 print(t.has_valid_recursive_sequence_lengths()) # True
W
wopeizl 已提交
490 491 492 493 494 495 496
           )DOC")
      .def("__getitem__", PySliceTensor, py::return_value_policy::reference,
           R"DOC(
           Slice the original Tensor, and remove the LoD information.

           Returns:
               out (Tensor): new Tensor(NOT LoDTensor).
497 498 499 500 501 502
           )DOC")
      .def("__str__", [](const LoDTensor &self) {
        std::stringstream ostr;
        ostr << self;
        return ostr.str();
      });
D
dangqingqing 已提交
503

Q
qijun 已提交
504 505 506 507 508 509 510 511 512 513 514
  py::class_<SelectedRows>(m, "SelectedRows")
      .def("__init__",
           [](SelectedRows &instance) { new (&instance) SelectedRows(); })
      .def("__init__",
           [](SelectedRows &instance, const std::vector<int64_t> rows,
              const int64_t &height) {
             new (&instance) SelectedRows(rows, height);
           })
      .def("get_tensor",
           [](SelectedRows &self) { return self.mutable_value(); },
           py::return_value_policy::reference)
515 516
      .def("numel",
           [](SelectedRows &self) -> int64_t { return self.value().numel(); })
Q
qijun 已提交
517 518
      .def("set_height", &SelectedRows::set_height)
      .def("height", &SelectedRows::height)
Q
qijun 已提交
519 520 521 522 523 524 525 526 527
      .def("set_rows",
           [](SelectedRows &self, std::vector<int64_t> rows) {
#ifndef PADDLE_WITH_CUDA
             self.set_rows(rows);
#else
        Vector<int64_t> new_rows(rows);
        self.set_rows(new_rows);
#endif
           })
528
      .def("sync_index", [](SelectedRows &instance) { instance.SyncIndex(); })
529
      .def("rows", [](SelectedRows &self) {
530 531 532 533 534
        auto rows = self.rows();
        std::vector<int64_t> new_rows;
        new_rows.reserve(rows.size());
        std::copy(rows.begin(), rows.end(), std::back_inserter(new_rows));
        return new_rows;
535
      });
Q
qijun 已提交
536

537
  py::class_<Variable>(m, "Variable", R"DOC(Variable Class.
538 539 540

All parameter, weight, gradient are variables in Paddle.
)DOC")
S
sneaxiy 已提交
541
      .def(py::init<>())
542
      .def("is_int", [](const Variable &var) { return var.IsType<int>(); })
543
      .def("set_int",
544 545
           [](Variable &var, int val) -> void { *var.GetMutable<int>() = val; })
      .def("get_int", [](const Variable &var) -> int { return var.Get<int>(); })
546 547 548 549 550 551 552
      .def("is_float", [](const Variable &var) { return var.IsType<float>(); })
      .def("set_float",
           [](Variable &var, float val) -> void {
             *var.GetMutable<float>() = val;
           })
      .def("get_float",
           [](const Variable &var) -> float { return var.Get<float>(); })
Y
Yu Yang 已提交
553
      .def("get_tensor",
554 555
           [](Variable &self) -> LoDTensor * {
             return self.GetMutable<LoDTensor>();
D
dangqingqing 已提交
556 557
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
558 559 560
      .def("get_lod_rank_table",
           [](Variable &self) { return self.GetMutable<LoDRankTable>(); },
           py::return_value_policy::reference)
Q
qijun 已提交
561 562 563 564 565
      .def("get_selected_rows",
           [](Variable &self) -> SelectedRows * {
             return self.GetMutable<SelectedRows>();
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
566 567 568
      .def("get_lod_tensor_array",
           [](Variable &self) { return self.GetMutable<LoDTensorArray>(); },
           py::return_value_policy::reference)
P
peizhilin 已提交
569
#if (defined(PADDLE_WITH_CUDA) && !defined(_WIN32))
D
Dong Zhihong 已提交
570 571 572 573 574
      .def("get_communicator",
           [](Variable &self) -> platform::Communicator * {
             return self.GetMutable<platform::Communicator>();
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
575
#endif
Y
Refine  
Yu Yang 已提交
576 577 578 579 580
      .def("get_reader",
           [](Variable &self) -> framework::ReaderHolder * {
             PADDLE_ENFORCE(self.IsType<framework::ReaderHolder>());
             return self.GetMutable<framework::ReaderHolder>();
           },
W
wopeizl 已提交
581
           py::return_value_policy::reference);
582

S
sneaxiy 已提交
583
  BindReader(&m);
Y
Refine  
Yu Yang 已提交
584

S
sneaxiy 已提交
585 586 587 588
  using LoDTensorBlockingQueue =
      ::paddle::operators::reader::LoDTensorBlockingQueue;
  using LoDTensorBlockingQueueHolder =
      ::paddle::operators::reader::LoDTensorBlockingQueueHolder;
S
sneaxiy 已提交
589

S
sneaxiy 已提交
590 591
  py::class_<LoDTensorBlockingQueue, std::shared_ptr<LoDTensorBlockingQueue>>(
      m, "LoDTensorBlockingQueue", "")
S
sneaxiy 已提交
592
      .def("push",
S
sneaxiy 已提交
593
           [](LoDTensorBlockingQueue &self,
S
sneaxiy 已提交
594
              const std::vector<framework::LoDTensor> &lod_tensor_vec) {
S
sneaxiy 已提交
595
             pybind11::gil_scoped_release release;
S
sneaxiy 已提交
596
             return self.Push(lod_tensor_vec);
S
sneaxiy 已提交
597
           })
S
sneaxiy 已提交
598 599 600 601
      .def("size", &LoDTensorBlockingQueue::Size)
      .def("capacity", &LoDTensorBlockingQueue::Cap)
      .def("close", &LoDTensorBlockingQueue::Close)
      .def("is_closed", &LoDTensorBlockingQueue::IsClosed);
S
sneaxiy 已提交
602

S
sneaxiy 已提交
603
  m.def("init_lod_tensor_blocking_queue",
Q
Qiao Longfei 已提交
604 605
        [](Variable &var,
           size_t capacity) -> std::shared_ptr<LoDTensorBlockingQueue> {
Q
Qiao Longfei 已提交
606
          VLOG(1) << "init_lod_tensor_blocking_queue";
Q
Qiao Longfei 已提交
607 608 609 610
          auto *holder = var.GetMutable<LoDTensorBlockingQueueHolder>();
          holder->InitOnce(capacity, FLAGS_reader_queue_speed_test_mode);
          return holder->GetQueue();
        },
S
sneaxiy 已提交
611
        py::return_value_policy::copy);
S
sneaxiy 已提交
612

S
sneaxiy 已提交
613
  py::class_<Scope>(m, "_Scope", R"DOC(
Q
Qiao Longfei 已提交
614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632
    Scope is an association of a name to Variable. All variables belong to Scope.

    Variables in a parent scope can be retrieved from local scope.

    You need to specify a scope to run a Net, i.e., `exe.Run(&scope)`.
    One net can run in different scopes and update different variable in the
    scope.

    You can create var in a scope and get it from the scope.

    Examples:
        .. code-block:: python

          # create tensor from a scope and set value to it.
          param = scope.var('Param').get_tensor()
          param_array = np.full((height, row_numel), 5.0).astype("float32")
          param.set(param_array, place)

        )DOC")
S
sneaxiy 已提交
633 634
      .def("_remove_from_pool",
           [](Scope &self) { ScopePool::Instance().Remove(&self); })
D
dongzhihong 已提交
635
      .def("var",
636
           [](Scope &self, const std::string &name) -> Variable * {
D
dongzhihong 已提交
637
             return self.Var(name);
Y
Yu Yang 已提交
638
           },
S
sneaxiy 已提交
639 640
           py::arg("name"),
           R"DOC(
641
           Find or create variable named :code:`name` in the current scope.
S
sneaxiy 已提交
642

643
           If the variable named :code:`name` does not exist in the
S
sneaxiy 已提交
644
           current scope, the variable would be created. Otherwise,
645
           return the existing variable.
S
sneaxiy 已提交
646 647

           Args:
648 649
               name (str): the variable name.

S
sneaxiy 已提交
650
           Returns:
651
               out (core.Variable): the found or created variable.
S
sneaxiy 已提交
652 653 654 655
           )DOC",
           py::return_value_policy::reference)
      .def("find_var", &Scope::FindVar, py::arg("name"),
           R"DOC(
656
           Find variable named :code:`name` in the current scope or
S
sneaxiy 已提交
657
           its parent scope. Return None if not found.
658

S
sneaxiy 已提交
659 660
           Args:
               name (str): the variable name.
661

S
sneaxiy 已提交
662
           Returns:
663
               out (core.Variable|None): the found variable or None.
S
sneaxiy 已提交
664
           )DOC",
665
           py::return_value_policy::reference)
666
      .def("new_scope", [](Scope &self) -> Scope * { return &self.NewScope(); },
S
sneaxiy 已提交
667 668 669 670 671 672
           R"DOC(
           Create a new sub-scope of the current scope.

           Returns:
               out (core._Scope): the created sub-scope.
           )DOC",
673
           py::return_value_policy::reference)
S
sneaxiy 已提交
674 675 676
      .def("drop_kids", &Scope::DropKids,
           R"DOC(
           Delete all sub-scopes of the current scope.
S
sneaxiy 已提交
677 678
           )DOC")
      .def("_kids", &Scope::kids);
679

S
sneaxiy 已提交
680 681 682 683 684 685
  m.def("Scope",
        []() -> Scope * {
          auto *s = new Scope();
          ScopePool::Instance().Insert(std::unique_ptr<Scope>(s));
          return s;
        },
S
sneaxiy 已提交
686 687
        R"DOC(
        Create a new scope.
688

S
sneaxiy 已提交
689 690 691
        Returns:
            out (core._Scope): the created scope.
        )DOC",
S
sneaxiy 已提交
692 693
        py::return_value_policy::reference);

Y
Yu Yang 已提交
694 695
  //! @note: Be careful! PyBind will return std::string as an unicode, not
  //! Python str. If you want a str object, you should cast them in Python.
Y
Yu Yang 已提交
696 697
  m.def("get_all_op_protos", []() -> std::vector<py::bytes> {
    std::vector<py::bytes> ret_values;
698 699 700 701 702 703 704 705 706 707
    for (auto &iter : OpInfoMap::Instance().map()) {
      auto &info = iter.second;
      if (info.HasOpProtoAndChecker()) {
        std::string str;
        PADDLE_ENFORCE(
            info.Proto().SerializeToString(&str),
            "Serialize OpProto Error. This could be a bug of Paddle.");
        ret_values.emplace_back(str);
      }
    }
Y
Yu Yang 已提交
708 709
    return ret_values;
  });
710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725
  m.def(
      "get_grad_op_desc", [](const OpDesc &op_desc,
                             const std::unordered_set<std::string> &no_grad_set,
                             const std::vector<BlockDesc *> &grad_sub_block) {
        std::unordered_map<std::string, std::string> grad_to_var;
        std::vector<std::unique_ptr<OpDesc>> grad_op_descs =
            framework::OpInfoMap::Instance()
                .Get(op_desc.Type())
                .GradOpMaker()(op_desc, no_grad_set, &grad_to_var,
                               grad_sub_block);
        std::vector<OpDesc *> grad_op_desc_ptrs(grad_op_descs.size());
        std::transform(grad_op_descs.begin(), grad_op_descs.end(),
                       grad_op_desc_ptrs.begin(),
                       [](std::unique_ptr<OpDesc> &p) { return p.release(); });
        return std::make_pair(grad_op_desc_ptrs, grad_to_var);
      });
Y
Yu Yang 已提交
726
  m.def("prune", [](const ProgramDesc &origin,
727
                    const std::vector<std::array<size_t, 2>> &targets) {
Y
Yu Yang 已提交
728
    ProgramDesc prog_with_targets(origin);
729
    for (const auto &t : targets) {
730
      prog_with_targets.MutableBlock(t[0])->Op(t[1])->SetIsTarget(true);
731
    }
732
    proto::ProgramDesc pruned_desc;
733
    Prune(*prog_with_targets.Proto(), &pruned_desc);
Y
Yu Yang 已提交
734
    return new ProgramDesc(pruned_desc);
735
  });
736 737 738 739
  m.def("empty_var_name",
        []() { return std::string(framework::kEmptyVarName); });
  m.def("grad_var_suffix",
        []() { return std::string(framework::kGradVarSuffix); });
740 741 742
  m.def_submodule(
       "var_names",
       "The module will return special predefined variable name in Paddle")
Y
Yi Wang 已提交
743 744
      .def("empty", []() { return kEmptyVarName; })
      .def("temp", []() { return kTempVarName; });
Q
qijun 已提交
745
  // clang-format off
Y
Yu Yang 已提交
746
  py::class_<paddle::platform::DeviceContext>(m, "DeviceContext")
Q
qijun 已提交
747 748
      .def_static("create",
                  [](paddle::platform::CPUPlace& place)
Q
qijun 已提交
749
                      -> paddle::platform::DeviceContext* {
Q
qijun 已提交
750 751 752
                    return new paddle::platform::CPUDeviceContext();
                  })
      .def_static("create",
D
dzhwinter 已提交
753
                  [](paddle::platform::CUDAPlace& place)
Q
qijun 已提交
754
                      -> paddle::platform::DeviceContext* {
755
#ifndef PADDLE_WITH_CUDA
D
dzhwinter 已提交
756
                    PADDLE_THROW("CUDAPlace is not supported in CPU device.");
Q
qijun 已提交
757
#else
Q
qijun 已提交
758
                    return new paddle::platform::CUDADeviceContext(place);
Q
qijun 已提交
759
#endif
C
chengduoZH 已提交
760 761 762 763 764 765 766 767 768 769 770
                  })
          .def_static("create",
                [](paddle::platform::CUDAPinnedPlace& place)
                        -> paddle::platform::DeviceContext* {
#ifndef PADDLE_WITH_CUDA
                  PADDLE_THROW(
                        "CUDAPinnedPlace is not supported in CPU device.");
#else
                  return new paddle::platform::CUDAPinnedDeviceContext(place);
#endif
                });;
D
Dong Zhihong 已提交
771
// clang-format on
P
peizhilin 已提交
772
#if (defined(PADDLE_WITH_CUDA) && !defined(_WIN32))
D
Dong Zhihong 已提交
773 774
  py::class_<platform::Communicator>(m, "Communicator").def(py::init<>());
#endif
775 776 777 778
  py::class_<platform::CUDAPlace>(m, "CUDAPlace", R"DOC(
    CUDAPlace is a descriptor of a device. It represents a GPU, and each CUDAPlace
    has a dev_id to indicate the number of cards represented by the current CUDAPlace.
    The memory of CUDAPlace with different dev_id is not accessible.
L
lujun 已提交
779 780 781 782 783 784

    Examples:
        .. code-block:: python

          gpu_place = fluid.CUDAPlace(0)

785
        )DOC")
S
sneaxiy 已提交
786 787 788 789 790 791 792 793 794 795 796 797
      .def("__init__",
           [](platform::CUDAPlace &self, int dev_id) {
#ifdef PADDLE_WITH_CUDA
             PADDLE_ENFORCE(
                 dev_id >= 0 && dev_id < platform::GetCUDADeviceCount(),
                 "Invalid CUDAPlace(%d), must inside [0, %d)", dev_id,
                 platform::GetCUDADeviceCount());
             new (&self) platform::CUDAPlace(dev_id);
#else
             PADDLE_THROW("Cannot use CUDAPlace in CPU only version");
#endif
           })
S
sneaxiy 已提交
798 799 800 801 802 803
      .def("_type", &PlaceIndex<platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::Place>)
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::CPUPlace>)
      .def("_equals",
           &IsSamePlace<platform::CUDAPlace, platform::CUDAPinnedPlace>)
D
dzhwinter 已提交
804
      .def("__str__", string::to_string<const platform::CUDAPlace &>);
Q
qijun 已提交
805

806 807 808
  py::class_<paddle::platform::CPUPlace>(m, "CPUPlace", R"DOC(
    CPUPlace is a descriptor of a device. It represents a CPU, and the memory
    CPUPlace can be accessed by CPU.
L
lujun 已提交
809 810 811 812 813 814

    Examples:
        .. code-block:: python

          cpu_place = fluid.CPUPlace()

815
        )DOC")
816
      .def(py::init<>())
S
sneaxiy 已提交
817 818 819 820 821 822
      .def("_type", &PlaceIndex<platform::CPUPlace>)
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::Place>)
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::CPUPlace>)
      .def("_equals",
           &IsSamePlace<platform::CPUPlace, platform::CUDAPinnedPlace>)
823
      .def("__str__", string::to_string<const platform::CPUPlace &>);
Y
Yu Yang 已提交
824

825 826 827
  py::class_<paddle::platform::CUDAPinnedPlace>(m, "CUDAPinnedPlace", R"DOC(
    CUDAPinnedPlace is a descriptor of a device. The memory of CUDAPinnedPlace
    can be accessed by GPU and CPU.
L
lujun 已提交
828 829 830 831 832 833

    Examples:
        .. code-block:: python

          place = fluid.CUDAPinnedPlace()

834
        )DOC")
S
sneaxiy 已提交
835
      .def("__init__",
S
sneaxiy 已提交
836
           [](platform::CUDAPinnedPlace &self) {
S
sneaxiy 已提交
837 838 839
#ifndef PADDLE_WITH_CUDA
             PADDLE_THROW("Cannot use CUDAPinnedPlace in CPU only version");
#endif
S
sneaxiy 已提交
840
             new (&self) platform::CUDAPinnedPlace();
S
sneaxiy 已提交
841
           })
S
sneaxiy 已提交
842 843 844 845 846 847 848 849
      .def("_type", &PlaceIndex<platform::CUDAPinnedPlace>)
      .def("_equals", &IsSamePlace<platform::CUDAPinnedPlace, platform::Place>)
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::CUDAPlace>)
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::CPUPlace>)
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::CUDAPinnedPlace>)
C
chengduoZH 已提交
850 851
      .def("__str__", string::to_string<const platform::CUDAPinnedPlace &>);

Y
Yu Yang 已提交
852 853
  py::class_<platform::Place>(m, "Place")
      .def(py::init<>())
S
sneaxiy 已提交
854 855 856 857 858
      .def("_type", &PlaceIndex<platform::Place>)
      .def("_equals", &IsSamePlace<platform::Place, platform::Place>)
      .def("_equals", &IsSamePlace<platform::Place, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::Place, platform::CPUPlace>)
      .def("_equals", &IsSamePlace<platform::Place, platform::CUDAPinnedPlace>)
X
xuezhong 已提交
859 860
      .def("is_gpu_place",
           [](platform::Place &self) { return platform::is_gpu_place(self); })
S
sneaxiy 已提交
861 862 863 864 865 866
      .def("is_cpu_place",
           [](platform::Place &self) { return platform::is_cpu_place(self); })
      .def("is_cuda_pinned_place",
           [](platform::Place &self) {
             return platform::is_cuda_pinned_place(self);
           })
X
xuezhong 已提交
867 868 869 870
      .def("gpu_device_id",
           [](platform::Place &self) {
             return boost::get<platform::CUDAPlace>(self).device;
           })
S
sneaxiy 已提交
871 872
      .def("set_place", [](platform::Place &self,
                           const platform::Place &other) { self = other; })
Y
Yu Yang 已提交
873 874 875 876 877
      .def("set_place",
           [](platform::Place &self, const platform::CPUPlace &cpu_place) {
             self = cpu_place;
           })
      .def("set_place",
D
dzhwinter 已提交
878
           [](platform::Place &self, const platform::CUDAPlace &gpu_place) {
Y
Yu Yang 已提交
879
             self = gpu_place;
C
chengduoZH 已提交
880 881
           })
      .def("set_place", [](platform::Place &self,
C
chengduoZH 已提交
882 883
                           const platform::CUDAPinnedPlace &cuda_pinned_place) {
        self = cuda_pinned_place;
C
chengduoZH 已提交
884
      });
Y
Yu Yang 已提交
885

Y
Yu Yang 已提交
886 887 888
  py::class_<OperatorBase>(m, "Operator")
      .def_static("create",
                  [](py::bytes protobin) {
889
                    proto::OpDesc desc;
Y
Yu Yang 已提交
890 891 892 893 894
                    PADDLE_ENFORCE(desc.ParsePartialFromString(protobin),
                                   "Cannot parse user input to OpDesc");
                    PADDLE_ENFORCE(desc.IsInitialized(),
                                   "User OpDesc is not initialized, reason %s",
                                   desc.InitializationErrorString());
895
                    return OpRegistry::CreateOp(desc);
Y
Yu Yang 已提交
896
                  })
897
      .def("run",
898
           [](OperatorBase &self, const Scope &scope,
D
dzhwinter 已提交
899 900 901
              const platform::CPUPlace &place) { self.Run(scope, place); })
      .def("run",
           [](OperatorBase &self, const Scope &scope,
D
dzhwinter 已提交
902
              const platform::CUDAPlace &place) { self.Run(scope, place); })
C
chengduoZH 已提交
903 904 905 906 907
      .def("run",
           [](OperatorBase &self, const Scope &scope,
              const platform::CUDAPinnedPlace &place) {
             self.Run(scope, place);
           })
Y
Yu Yang 已提交
908 909 910 911 912 913 914
      .def("type",
           [](const OperatorBase &op) -> std::string { return op.Type(); })
      .def("outputs",
           [](const OperatorBase &op)
               -> std::map<std::string, std::vector<std::string>> {
                 return op.Outputs();
               })
Q
qijun 已提交
915 916
      .def("output_vars",
           [](const OperatorBase &op) { return op.OutputVars(true); })
Y
Yu Yang 已提交
917
      .def("inputs", [](const OperatorBase &op) { return op.Inputs(); })
Q
qijun 已提交
918
      .def("input_vars", [](const OperatorBase &op) { return op.InputVars(); })
Y
Yu Yang 已提交
919 920 921 922
      .def("__str__", &OperatorBase::DebugString)
      .def("no_intermediate_outputs",
           [](const OperatorBase &op) { return op.OutputVars(false); })
      .def("support_gpu", &OperatorBase::SupportGPU);
Y
Yu Yang 已提交
923

924 925 926
  py::class_<framework::ExecutorPrepareContext>(m, "ExecutorPrepareContext")
      .def(py::init<const ProgramDesc &, size_t>());

F
fengjiayi 已提交
927
  py::class_<framework::Executor>(m, "Executor")
D
dzhwinter 已提交
928
      .def(py::init<const platform::Place &>())
Y
Yancey1989 已提交
929
      .def("close", &Executor::Close)
930 931 932 933 934 935 936 937 938 939 940 941 942 943
      .def("run_from_dataset", &Executor::RunFromDataset,
           py::call_guard<py::gil_scoped_release>())
      .def("run_prepared_ctx",
           [](Executor &self, ExecutorPrepareContext *ctx, Scope *scope,
              std::map<std::string, const LoDTensor *> *feed_targets,
              std::map<std::string, LoDTensor *> *fetch_targets,
              bool create_local_scope = true, bool create_vars = true,
              const std::string &feed_holder_name = "feed",
              const std::string &fetch_holder_name = "fetch") {
             pybind11::gil_scoped_release release;
             self.RunPreparedContext(ctx, scope, feed_targets, fetch_targets,
                                     create_local_scope, create_vars,
                                     feed_holder_name, fetch_holder_name);
           })
G
guru4elephant 已提交
944 945 946 947 948 949 950 951
      .def("run_cached_prepared_ctx",
           [](Executor &self, ExecutorPrepareContext *ctx, Scope *scope,
              bool create_local_scope = true, bool create_vars = true,
              bool keep_kids = false) {
             pybind11::gil_scoped_release release;
             self.RunPreparedContext(ctx, scope, create_local_scope,
                                     create_vars, keep_kids);
           })
952 953
      .def("prepare_ctx_cache", &Executor::PrepareCtxCache,
           py::call_guard<py::gil_scoped_release>())
954 955
      .def("create_variables", &Executor::CreateVariables,
           py::call_guard<py::gil_scoped_release>())
S
sneaxiy 已提交
956
      .def("run", [](Executor &self, const ProgramDesc &prog, Scope *scope,
S
sneaxiy 已提交
957 958
                     int block_id, bool create_local_scope, bool create_vars,
                     const std::vector<std::string> &fetch_vars) {
S
sneaxiy 已提交
959
        pybind11::gil_scoped_release release;
S
sneaxiy 已提交
960 961
        self.Run(prog, scope, block_id, create_local_scope, create_vars,
                 fetch_vars);
S
sneaxiy 已提交
962
      });
S
sneaxiy 已提交
963

D
dzhwinter 已提交
964
  m.def("init_gflags", framework::InitGflags);
Y
Yang Yu 已提交
965
  m.def("init_glog", framework::InitGLOG);
966
  m.def("init_dgc", framework::InitDGC);
X
Xin Pan 已提交
967 968
  m.def("init_devices",
        [](bool init_p2p) { framework::InitDevices(init_p2p); });
969

970
  m.def("is_compiled_with_ngraph", IsCompiledWithNGRAPH);
971
  m.def("is_compiled_with_cuda", IsCompiledWithCUDA);
972
  m.def("is_compiled_with_mkldnn", IsCompiledWithMKLDNN);
973
  m.def("is_compiled_with_brpc", IsCompiledWithBrpc);
Y
update  
Yancey1989 已提交
974
  m.def("is_compiled_with_dist", IsCompiledWithDIST);
975 976 977 978 979 980
#ifdef PADDLE_WITH_CUDA
  m.def("is_float16_supported", [](const platform::CUDAPlace &place) -> bool {
    // Only GPUs with Compute Capability >= 53 support float16
    return platform::GetCUDAComputeCapability(place.device) >= 53;
  });
#endif
981

982
  m.def("set_feed_variable", framework::SetFeedVariable);
Q
qijun 已提交
983
  m.def("get_fetch_variable", framework::GetFetchVariable);
984
  m.def("get_variable_tensor", framework::GetVariableTensor);
Q
qijun 已提交
985

X
Xin Pan 已提交
986 987
  m.def("_is_program_version_supported", IsProgramVersionSupported);

988 989 990 991 992
  BindProgramDesc(&m);
  BindBlockDesc(&m);
  BindVarDsec(&m);
  BindOpDesc(&m);
  BindConstValue(&m);
Y
Yu Yang 已提交
993

Y
Yu Yang 已提交
994 995 996 997 998 999 1000 1001 1002
  py::class_<framework::LoDRankTable>(m, "LodRankTable")
      .def("items", [](framework::LoDRankTable &table) {
        std::vector<std::pair<size_t, size_t>> res;
        for (auto &item : table.items()) {
          res.push_back({item.index, item.length});
        }
        return res;
      });

Z
Zeng Jinle 已提交
1003 1004 1005 1006 1007
  py::class_<LoDTensorArray>(m, "LoDTensorArray", R"DOC(
    Array of LoDTensor.

    Examples:
        .. code-block:: python
1008

Z
Zeng Jinle 已提交
1009 1010 1011 1012
          import paddle.fluid as fluid

          arr = fluid.LoDTensorArray()
)DOC")
S
sneaxiy 已提交
1013 1014
      .def("__init__",
           [](LoDTensorArray &instance) { new (&instance) LoDTensorArray(); })
Y
Yu Yang 已提交
1015 1016 1017 1018 1019 1020 1021 1022 1023 1024
      .def("__getitem__",
           [](LoDTensorArray &self, size_t i) { return &self.at(i); },
           py::return_value_policy::reference)
      .def("__len__", [](LoDTensorArray &self) { return self.size(); })
      .def("__setitem__",
           [](LoDTensorArray &self, size_t i, const LoDTensor &t) {
             PADDLE_ENFORCE_LT(i, self.size());
             self[i].ShareDataWith(t);
             self[i].set_lod(t.lod());
           })
S
sneaxiy 已提交
1025 1026 1027 1028 1029 1030
      .def("append",
           [](LoDTensorArray &self, const LoDTensor &t) {
             self.emplace_back();
             self.back().ShareDataWith(t);
             self.back().set_lod(t.lod());
           },
Z
Zeng Jinle 已提交
1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044
           py::arg("tensor"), R"DOC(
             Append a LoDensor to LoDTensorArray.

             Examples:
                 .. code-block:: python

                   import paddle.fluid as fluid
                   import numpy as np

                   arr = fluid.LoDTensorArray()
                   t = fluid.LoDTensor()
                   t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                   arr.append(t)
           )DOC");
Y
Yu Yang 已提交
1045

D
dzhwinter 已提交
1046 1047 1048
  m.def("IsInplace",
        [](std::string op) -> bool { return operators::IsInplace(op); });

Y
Yu Yang 已提交
1049
  m.def("op_support_gpu", OpSupportGPU);
D
Dong Zhihong 已提交
1050
#ifdef PADDLE_WITH_CUDA
D
Dong Zhihong 已提交
1051
  m.def("get_cuda_device_count", platform::GetCUDADeviceCount);
D
dangqingqing 已提交
1052

P
peizhilin 已提交
1053
#ifndef _WIN32
D
dangqingqing 已提交
1054 1055 1056
  m.def("nvprof_init", platform::CudaProfilerInit);
  m.def("nvprof_start", platform::CudaProfilerStart);
  m.def("nvprof_stop", platform::CudaProfilerStop);
D
Dong Zhihong 已提交
1057
#endif
P
peizhilin 已提交
1058
#endif
Y
Yu Yang 已提交
1059

1060 1061 1062 1063
  py::enum_<platform::ProfilerState>(m, "ProfilerState", py::arithmetic())
      .value("kDisabled", platform::ProfilerState::kDisabled)
      .value("kCPU", platform::ProfilerState::kCPU)
      .value("kCUDA", platform::ProfilerState::kCUDA)
1064
      .value("kAll", platform::ProfilerState::kAll)
1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077
      .export_values();

  py::enum_<platform::EventSortingKey>(m, "EventSortingKey", py::arithmetic())
      .value("kDefault", platform::EventSortingKey::kDefault)
      .value("kCalls", platform::EventSortingKey::kCalls)
      .value("kTotal", platform::EventSortingKey::kTotal)
      .value("kMin", platform::EventSortingKey::kMin)
      .value("kMax", platform::EventSortingKey::kMax)
      .value("kAve", platform::EventSortingKey::kAve)
      .export_values();

  m.def("enable_profiler", platform::EnableProfiler);
  m.def("disable_profiler", platform::DisableProfiler);
X
Xin Pan 已提交
1078
  m.def("is_profiler_enabled", platform::IsProfileEnabled);
1079
  m.def("reset_profiler", platform::ResetProfiler);
1080
  m.def("get_pass", [](const std::string &pass_type) {
W
WangZhen 已提交
1081 1082 1083
    auto pass = framework::ir::PassRegistry::Instance().Get(pass_type);
    return std::shared_ptr<framework::ir::Pass>(std::move(pass));
  });
Y
Yu Yang 已提交
1084

1085 1086
  py::class_<ir::Pass, std::shared_ptr<ir::Pass>> pass(m, "Pass");
  pass.def(py::init())
W
WangZhen 已提交
1087
      .def("has", &ir::Pass::Has)
1088 1089 1090
      .def("set_not_owned",
           [](ir::Pass &self, const std::string &attr_name, ProgramDesc &attr) {
             self.SetNotOwned<ProgramDesc>(attr_name, &attr);
W
WangZhen 已提交
1091
           })
1092
      .def(
1093
          "set",
1094 1095 1096
          [](ir::Pass &self, const std::string &name, const std::string &attr) {
            self.Set<std::string>(name, new std::string(attr));
          })
1097 1098
      .def("set", [](ir::Pass &self, const std::string &name,
                     int val) { self.Set<const int>(name, new int(val)); })
F
flame 已提交
1099 1100
      .def("type", &ir::Pass::Type)
      .def("apply", [](ir::Pass &self, std::shared_ptr<ir::Graph> graph) {
1101
        self.Apply(graph.get());
F
flame 已提交
1102
      });
1103

X
fix  
Xin Pan 已提交
1104 1105
  py::class_<ir::PassBuilder, std::shared_ptr<ir::PassBuilder>> pb(
      m, "PassBuilder");
1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119
  pb.def(py::init())
      .def("append_pass",
           [](ir::PassBuilder &self,
              const std::string &pass_type) -> std::shared_ptr<ir::Pass> {
             return self.AppendPass(pass_type);
           })
      .def("all_passes", [](ir::PassBuilder &self) { return self.AllPasses(); })
      .def("insert_pass",
           [](ir::PassBuilder &self, size_t idx, const std::string &pass_type) {
             return self.InsertPass(idx, pass_type);
           })
      .def("remove_pass",
           [](ir::PassBuilder &self, size_t idx) { self.RemovePass(idx); });

Y
yuyang18 已提交
1120
  // -- python binds for parallel executor.
X
Xin Pan 已提交
1121

Y
yuyang18 已提交
1122
  py::class_<ParallelExecutor> pe(m, "ParallelExecutor");
C
chengduo 已提交
1123 1124 1125 1126
  py::class_<ExecutionStrategy> exec_strategy(pe, "ExecutionStrategy", R"DOC(
    ExecutionStrategy allows the user to more preciously control how to run
    the program in ParallelExecutor by setting the property.

C
chengduo 已提交
1127 1128 1129
    Examples:
        .. code-block:: python

1130 1131 1132 1133 1134 1135 1136 1137 1138 1139
          x = fluid.layers.data(name='x', shape=[13], dtype='float32')
          y = fluid.layers.data(name='y', shape=[1], dtype='float32')
          y_predict = fluid.layers.fc(input=x, size=1, act=None)

          cost = fluid.layers.square_error_cost(input=y_predict, label=y)
          avg_loss = fluid.layers.mean(cost)

          sgd_optimizer = fluid.optimizer.SGD(learning_rate=0.001)
          sgd_optimizer.minimize(avg_loss)

C
chengduo 已提交
1140 1141 1142
          exec_strategy = fluid.ExecutionStrategy()
          exec_strategy.num_threads = 4

1143 1144
          train_exe = fluid.ParallelExecutor(use_cuda=False,
                                             loss_name=avg_loss.name,
C
chengduo 已提交
1145 1146
                                             exec_strategy=exec_strategy)

C
chengduo 已提交
1147 1148
        )DOC");

Y
yuyang18 已提交
1149
  exec_strategy.def(py::init())
Y
yuyang18 已提交
1150 1151 1152 1153 1154
      .def_property(
          "num_threads",
          [](const ExecutionStrategy &self) { return self.num_threads_; },
          [](ExecutionStrategy &self, size_t num_threads) {
            self.num_threads_ = num_threads;
C
chengduo 已提交
1155 1156 1157 1158 1159 1160 1161 1162 1163 1164
          },
          R"DOC(The type is INT, num_threads represents the size of thread pool that
            used to run the operators of the current program in ParallelExecutor.
            If :math:`num\_threads=1`, all the operators will execute one by one,
            but the order maybe difference between iterations.
            If it is not set, it will be set in ParallelExecutor according to the
            device type and device count, for GPU, :math:`num\_threads=device\_count*4`, for CPU,
            :math:`num\_threads=CPU\_NUM*4`, the explanation of:math:`CPU\_NUM` is in ParallelExecutor.
            if it is not set, ParallelExecutor will get the cpu count by calling
            `multiprocessing.cpu_count()`. Default 0.)DOC")
Y
yuyang18 已提交
1165
      .def_property(
1166 1167 1168 1169
          "use_cuda",
          [](const ExecutionStrategy &self) { return self.use_cuda_; },
          [](ExecutionStrategy &self, bool use_cuda) {
            self.use_cuda_ = use_cuda;
C
chengduo 已提交
1170 1171 1172 1173
          })  // FIXME(chengduo): Doesn't add doc for 'use_cuda', use_cuda may
      // make user confuse, because ParallelExecutor has a parameter named
      // 'use_cuda' too, in current implementation, ParallelExecutor's
      // 'use_cuda' will rewrite ExecutionStrategy's 'use_cuda'.
Y
yuyang18 已提交
1174 1175 1176 1177 1178
      .def_property(
          "allow_op_delay",
          [](const ExecutionStrategy &self) { return self.allow_op_delay_; },
          [](ExecutionStrategy &self, bool allow_op_delay) {
            self.allow_op_delay_ = allow_op_delay;
C
chengduo 已提交
1179 1180 1181 1182
          },
          R"DOC(The type is BOOL, allow_op_delay represents whether to delay the
                communication operators to run, it may make the execution faster.
                Note that in some models, allow_op_delay may cause program hang. Default False.)DOC")
Y
yuyang18 已提交
1183 1184 1185 1186 1187 1188 1189
      .def_property(
          "num_iteration_per_drop_scope",
          [](const ExecutionStrategy &self) {
            return self.num_iteration_per_drop_scope_;
          },
          [](ExecutionStrategy &self, size_t num_iteration_per_drop_scope) {
            self.num_iteration_per_drop_scope_ = num_iteration_per_drop_scope;
C
chengduo 已提交
1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200
          },
          R"DOC(The type is INT, num_iteration_per_drop_scope indicates how
                many iterations to clean up the temp variables which
                is generated during execution. It may make the execution faster,
                because the temp variable's shape maybe the same between two iterations. Default 100.

                NOTES:
                    1. If you fetch data when calling the 'run', the ParallelExecutor
                       will clean up the temp variables at the end of the current iteration.
                    2. In some NLP model, it may cause the GPU memory is insufficient,
                       in this case, you should reduce `num_iteration_per_drop_scope`.
1201
              )DOC")
Q
Qiao Longfei 已提交
1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212
      .def_property(
          "num_iteration_per_run",
          [](const ExecutionStrategy &self) {
            return self.num_iteration_per_run_;
          },
          [](ExecutionStrategy &self, size_t num_iteration_per_run) {
            self.num_iteration_per_run_ = num_iteration_per_run;
          },
          R"DOC(This config that how many iteration the executor will run when
                user call pe.run() in python
              )DOC")
1213 1214 1215 1216 1217
      .def_property("_dry_run",
                    [](const ExecutionStrategy &self) { return self.dry_run_; },
                    [](ExecutionStrategy &self, bool dry_run) {
                      self.dry_run_ = dry_run;
                    });
C
chengduo 已提交
1218

Y
yuyang18 已提交
1219
  exec_strategy.def_property(
Y
yuyang18 已提交
1220 1221 1222 1223 1224 1225 1226
      "use_experimental_executor",
      [](const ExecutionStrategy &self) {
        return self.type_ == ExecutionStrategy::kExperimental;
      },
      [](ExecutionStrategy &self, bool experimental) {
        self.type_ = experimental ? ExecutionStrategy::kExperimental
                                  : ExecutionStrategy::kDefault;
Y
yuyang18 已提交
1227 1228
      });

C
chengduo 已提交
1229 1230 1231 1232
  py::class_<BuildStrategy> build_strategy(pe, "BuildStrategy", R"DOC(
    BuildStrategy allows the user to more preciously control how to
    build the SSA Graph in ParallelExecutor by setting the property.

C
chengduo 已提交
1233 1234 1235
    Examples:
        .. code-block:: python

F
flame 已提交
1236 1237 1238
            import paddle.fluid as fluid
            build_strategy = fluid.BuildStrategy()
            build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce
C
chengduo 已提交
1239
)DOC");
Y
yuyang18 已提交
1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255

  py::enum_<BuildStrategy::ReduceStrategy>(build_strategy, "ReduceStrategy")
      .value("Reduce", BuildStrategy::ReduceStrategy::kReduce)
      .value("AllReduce", BuildStrategy::ReduceStrategy::kAllReduce);
  py::enum_<BuildStrategy::GradientScaleStrategy>(build_strategy,
                                                  "GradientScaleStrategy")
      .value("CoeffNumDevice",
             BuildStrategy::GradientScaleStrategy::kCoeffNumDevice)
      .value("One", BuildStrategy::GradientScaleStrategy::kOne)
      .value("Customized", BuildStrategy::GradientScaleStrategy::kCustomized);

  build_strategy.def(py::init())
      .def_property(
          "reduce_strategy",
          [](const BuildStrategy &self) { return self.reduce_; },
          [](BuildStrategy &self, BuildStrategy::ReduceStrategy strategy) {
X
Xin Pan 已提交
1256
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
Y
yuyang18 已提交
1257
            self.reduce_ = strategy;
C
chengduo 已提交
1258 1259
          },
          R"DOC(The type is STR, there are two reduce strategies in ParallelExecutor,
F
flame 已提交
1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272
                'AllReduce' and 'Reduce'. If you want that all the parameters'
                optimization are done on all devices independently, you should choose 'AllReduce';
                if you choose 'Reduce', all the parameters' optimization will be evenly distributed
                to different devices, and then broadcast the optimized parameter to other devices.
                In some models, `Reduce` is faster. Default 'AllReduce'.

                Examples:
                    .. code-block:: python

                        import paddle.fluid as fluid
                        build_strategy = fluid.BuildStrategy()
                        build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce
                  )DOC")
Y
yuyang18 已提交
1273 1274 1275 1276 1277
      .def_property(
          "gradient_scale_strategy",
          [](const BuildStrategy &self) { return self.gradient_scale_; },
          [](BuildStrategy &self,
             BuildStrategy::GradientScaleStrategy strategy) {
X
Xin Pan 已提交
1278
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
Y
yuyang18 已提交
1279
            self.gradient_scale_ = strategy;
C
chengduo 已提交
1280 1281
          },
          R"DOC(The type is STR, there are three ways of defining :math:`loss@grad` in
F
flame 已提交
1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293
                ParallelExecutor, 'CoeffNumDevice', 'One' and 'Customized'. By default,
                ParallelExecutor sets the :math:`loss@grad` according to the number of devices.
                If you want to customize :math:`loss@grad`, you can choose 'Customized'.
                Default 'CoeffNumDevice'.

                Examples:
                    .. code-block:: python

                        import paddle.fluid as fluid
                        build_strategy = fluid.BuildStrategy()
                        build_strategy.gradient_scale_strategy = True
                   )DOC")
Y
yuyang18 已提交
1294 1295 1296 1297
      .def_property(
          "debug_graphviz_path",
          [](const BuildStrategy &self) { return self.debug_graphviz_path_; },
          [](BuildStrategy &self, const std::string &path) {
X
Xin Pan 已提交
1298
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
Y
yuyang18 已提交
1299
            self.debug_graphviz_path_ = path;
C
chengduo 已提交
1300 1301
          },
          R"DOC(The type is STR, debug_graphviz_path indicate the path that
F
flame 已提交
1302 1303 1304 1305 1306 1307 1308 1309 1310 1311
                writing the SSA Graph to file in the form of graphviz.
                It is useful for debugging. Default ""

                Examples:
                    .. code-block:: python

                        import paddle.fluid as fluid
                        build_strategy = fluid.BuildStrategy()
                        build_strategy.debug_graphviz_path = ""
                    )DOC")
S
sneaxiy 已提交
1312 1313 1314 1315 1316 1317
      .def_property(
          "enable_sequential_execution",
          [](const BuildStrategy &self) {
            return self.enable_sequential_execution_;
          },
          [](BuildStrategy &self, bool b) {
X
Xin Pan 已提交
1318
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
S
sneaxiy 已提交
1319 1320
            self.enable_sequential_execution_ = b;
          },
F
flame 已提交
1321 1322 1323 1324 1325 1326 1327 1328 1329
          R"DOC(The type is BOOL. If set True, the execution order of ops would be the same as what is in the program. Default False.

                Examples:
                    .. code-block:: python

                        import paddle.fluid as fluid
                        build_strategy = fluid.BuildStrategy()
                        build_strategy.enable_sequential_execution = True
          )DOC")
S
sneaxiy 已提交
1330 1331 1332 1333 1334 1335
      .def_property(
          "remove_unnecessary_lock",
          [](const BuildStrategy &self) {
            return self.remove_unnecessary_lock_;
          },
          [](BuildStrategy &self, bool b) {
X
Xin Pan 已提交
1336
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
S
sneaxiy 已提交
1337 1338
            self.remove_unnecessary_lock_ = b;
          },
F
flame 已提交
1339 1340 1341 1342 1343 1344 1345 1346 1347
          R"DOC(The type is BOOL. If set True, some locks in GPU ops would be released and ParallelExecutor would run faster. Default True.

                Examples:
                    .. code-block:: python

                        import paddle.fluid as fluid
                        build_strategy = fluid.BuildStrategy()
                        build_strategy.remove_unnecessary_lock = True
          )DOC")
1348 1349 1350 1351
      .def_property(
          "num_trainers",
          [](const BuildStrategy &self) { return self.num_trainers_; },
          [](BuildStrategy &self, int num_trainers) {
1352 1353 1354
#ifdef WIN32
            PADDLE_THROW("Windows has NO support to distribute mode.");
#endif
1355 1356
            self.num_trainers_ = num_trainers;
          })
1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368
      .def_property(
          "trainers_endpoints",
          [](const BuildStrategy &self) { return self.trainers_endpoints_; },
          [](BuildStrategy &self,
             const std::vector<std::string> &trainers_endpoints) {
            self.trainers_endpoints_ = trainers_endpoints;
          })
      .def_property("trainer_id",
                    [](const BuildStrategy &self) { return self.trainer_id_; },
                    [](BuildStrategy &self, int trainer_id) {
                      self.trainer_id_ = trainer_id;
                    })
1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396
      .def_property(
          "nccl_comm_num",
          [](const BuildStrategy &self) { return self.nccl_comm_num_; },
          [](BuildStrategy &self, int nccl_comm_num) {
            self.nccl_comm_num_ = nccl_comm_num;
          })
      .def_property("use_hierarchical_allreduce_",
                    [](const BuildStrategy &self) {
                      return self.use_hierarchical_allreduce_;
                    },
                    [](BuildStrategy &self, bool use) {
                      self.use_hierarchical_allreduce_ = use;
                    })
      .def_property("hierarchical_allreduce_inter_nranks_",
                    [](const BuildStrategy &self) {
                      return self.hierarchical_allreduce_inter_nranks_;
                    },
                    [](BuildStrategy &self, int nranks) {
                      self.hierarchical_allreduce_inter_nranks_ = nranks;
                    })
      .def_property("hierarchical_allreduce_exter_nranks_",
                    [](const BuildStrategy &self) {
                      return self.hierarchical_allreduce_exter_nranks_;
                    },
                    [](BuildStrategy &self, int nranks) {
                      self.hierarchical_allreduce_exter_nranks_ = nranks;
                    })

C
chengduo 已提交
1397 1398 1399 1400 1401 1402
      .def_property(
          "fuse_elewise_add_act_ops",
          [](const BuildStrategy &self) {
            return self.fuse_elewise_add_act_ops_;
          },
          [](BuildStrategy &self, bool b) {
X
Xin Pan 已提交
1403
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
C
chengduo 已提交
1404 1405 1406
            self.fuse_elewise_add_act_ops_ = b;
          },
          R"DOC(The type is BOOL, fuse_elewise_add_act_ops indicate whether
F
flame 已提交
1407 1408 1409 1410 1411 1412 1413 1414 1415 1416
                to fuse elementwise_add_op and activation_op,
                it may make the execution faster. Default False

                Examples:
                    .. code-block:: python

                        import paddle.fluid as fluid
                        build_strategy = fluid.BuildStrategy()
                        build_strategy.fuse_elewise_add_act_ops = True
                     )DOC")
1417 1418 1419 1420 1421 1422 1423 1424 1425 1426
      .def_property(
          "fuse_relu_depthwise_conv",
          [](const BuildStrategy &self) {
            return self.fuse_relu_depthwise_conv_;
          },
          [](BuildStrategy &self, bool b) {
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
            self.fuse_relu_depthwise_conv_ = b;
          },
          R"DOC(The type is BOOL, fuse_relu_depthwise_conv indicate whether
F
flame 已提交
1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438
                to fuse relu and depthwise_conv2d,
                it will save GPU memory and may make the execution faster.
                This options is only available in GPU devices.
                Default False.

                Examples:
                    .. code-block:: python

                        import paddle.fluid as fluid
                        build_strategy = fluid.BuildStrategy()
                        build_strategy.fuse_relu_depthwise_conv = True
          )DOC")
1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451
      .def_property(
          "fuse_broadcast_ops",
          [](const BuildStrategy &self) { return self.fuse_broadcast_ops_; },
          [](BuildStrategy &self, bool b) {
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
            self.fuse_broadcast_ops_ = b;
          },
          R"DOC(The type is BOOL, fuse_broadcast_op indicates whether
                      to fuse the broadcast ops. Note that, in Reduce mode,
                      fusing broadcast ops may make the program faster. Because
                      fusing broadcast OP equals delaying the execution of all
                      broadcast Ops, in this case, all nccl streams are used only
                      for NCCLReduce operations for a period of time. Default False.)DOC")
C
chengduo 已提交
1452 1453 1454 1455 1456 1457 1458 1459 1460
      .def_property("fuse_all_optimizer_ops",
                    [](const BuildStrategy &self) {
                      return self.fuse_all_optimizer_ops_;
                    },
                    [](BuildStrategy &self, bool b) {
                      PADDLE_ENFORCE(!self.IsFinalized(),
                                     "BuildStrategy is finlaized.");
                      self.fuse_all_optimizer_ops_ = b;
                    })
Q
qingqing01 已提交
1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474
      .def_property(
          "sync_batch_norm",
          [](const BuildStrategy &self) { return self.sync_batch_norm_; },
          [](BuildStrategy &self, bool b) {
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
            self.sync_batch_norm_ = b;
          },
          R"DOC(The type is BOOL, sync_batch_norm indicates whether to use
                synchronous batch normalization which synchronizes the mean
                and variance through multi-devices in training phase.

                Current implementation doesn't support FP16 training and CPU.
                And only synchronous on one machine, not all machines.

F
flame 已提交
1475 1476 1477 1478 1479 1480 1481 1482 1483
                Default False

                Examples:
                    .. code-block:: python

                        import paddle.fluid as fluid
                        build_strategy = fluid.BuildStrategy()
                        build_strategy.sync_batch_norm = True
                )DOC")
D
dzhwinter 已提交
1484 1485 1486
      .def_property(
          "memory_optimize",
          [](const BuildStrategy &self) { return self.memory_optimize_; },
1487
          [](BuildStrategy &self, bool b) { self.memory_optimize_ = b; },
1488
          R"DOC(The type is BOOL, memory opitimize aims to save total memory
1489
                consumption, set to True to enable it.
1490 1491

                Memory Optimize is our experimental feature, some variables
1492 1493 1494
                may be reused/removed by optimize strategy. If you need to
                fetch some variable values when using this feature, please
                set the persistable property of the variables to True.
1495

1496
                Default False)DOC")
1497 1498 1499
      .def_property(
          "is_distribution",
          [](const BuildStrategy &self) { return self.is_distribution_; },
1500 1501 1502 1503 1504 1505 1506 1507 1508
          [](BuildStrategy &self, bool b) {
#ifdef WIN32
            if (b) {
              PADDLE_THROW("Windows has NO support to distribute mode.");
            }
#else
            self.is_distribution_ = b;
#endif
          })
Q
can run  
Qiao Longfei 已提交
1509 1510 1511
      .def_property("async_mode",
                    [](const BuildStrategy &self) { return self.async_mode_; },
                    [](BuildStrategy &self, bool b) { self.async_mode_ = b; })
D
dzhwinter 已提交
1512
      .def_property(
D
dzhwinter 已提交
1513 1514 1515
          "enable_inplace",
          [](const BuildStrategy &self) { return self.enable_inplace_; },
          [](BuildStrategy &self, bool b) { self.enable_inplace_ = b; })
C
chengduo 已提交
1516 1517 1518 1519
      .def_property(
          "fuse_all_reduce_ops",
          [](const BuildStrategy &self) { return self.fuse_all_reduce_ops_; },
          [](BuildStrategy &self, bool b) { self.fuse_all_reduce_ops_ = b; })
1520 1521 1522 1523 1524 1525 1526
      .def_property("enable_backward_optimizer_op_deps",
                    [](const BuildStrategy &self) {
                      return self.enable_backward_optimizer_op_deps_;
                    },
                    [](BuildStrategy &self, bool b) {
                      self.enable_backward_optimizer_op_deps_ = b;
                    })
1527 1528 1529 1530
      .def_property(
          "cache_runtime_context",
          [](const BuildStrategy &self) { return self.cache_runtime_context_; },
          [](BuildStrategy &self, bool b) { self.cache_runtime_context_ = b; })
1531 1532 1533 1534 1535 1536 1537 1538 1539
      .def_property(
          "mkldnn_enabled_op_types",
          [](const BuildStrategy &self) {
            return self.mkldnn_enabled_op_types_;
          },
          [](BuildStrategy &self,
             const std::unordered_set<std::string> &mkldnn_enabled_op_types) {
            self.mkldnn_enabled_op_types_ = mkldnn_enabled_op_types;
          })
1540
      .def("_finalize_strategy_and_create_passes",
X
fix  
Xin Pan 已提交
1541
           [](BuildStrategy &self) -> std::shared_ptr<ir::PassBuilder> {
1542 1543 1544 1545 1546
             return self.CreatePassesFromStrategy(true);
           },
           R"DOC(Allow user to customized passes. Normally model-specific
                optimization passes should be defined in this way. BuildStrategy
                cannot be updated after being finalized.)DOC");
Y
yuyang18 已提交
1547 1548

  pe.def(py::init<const std::vector<platform::Place> &,
Y
Yan Xu 已提交
1549
                  const std::vector<std::string> &, const std::string &,
X
Xin Pan 已提交
1550
                  Scope *, std::vector<Scope *> &, const ExecutionStrategy &,
X
Xin Pan 已提交
1551
                  const BuildStrategy &, ir::Graph *>())
Y
Yu Yang 已提交
1552 1553 1554 1555
      // NOTE: even we return a vec<Scope*>* to Python use reference policy.
      // We still cannot get local_scope from this vector, since the element
      // of vec<Scope*> will be freed by Python GC. We can only return Scope*
      // one by one and mark them as reference.
1556 1557 1558 1559 1560
      .def("local_scopes",
           [](ParallelExecutor &self) -> std::vector<Scope *> * {
             return &self.GetLocalScopes();
           },
           py::return_value_policy::reference)
1561 1562 1563
      .def("drop_local_exe_scopes", &ParallelExecutor::DropLocalExeScopes)
      .def("_need_create_local_exe_scopes",
           &ParallelExecutor::NeedCreateLocalExeScope)
Y
Yu Yang 已提交
1564 1565 1566 1567
      .def("feed_tensors_into_local_scopes",
           &ParallelExecutor::FeedTensorsIntoLocalScopes)
      .def("feed_and_split_tensor_into_local_scopes",
           &ParallelExecutor::FeedAndSplitTensorIntoLocalScopes)
S
sneaxiy 已提交
1568 1569 1570 1571 1572 1573
      .def("run", [](ParallelExecutor &self,
                     const std::vector<std::string> &fetch_tensors,
                     const std::string &fetched_var_name) {
        pybind11::gil_scoped_release release;
        self.Run(fetch_tensors, fetched_var_name);
      });
Y
Yu Yang 已提交
1574

1575
  BindRecordIOWriter(&m);
W
Wang Guibao 已提交
1576
  BindAsyncExecutor(&m);
D
dongdaxiang 已提交
1577
  BindFleetWrapper(&m);
W
wopeizl 已提交
1578
#ifndef _WIN32
D
dongdaxiang 已提交
1579
  BindNCCLWrapper(&m);
W
wopeizl 已提交
1580
#endif
F
flame 已提交
1581 1582
  BindGraph(&m);
  BindNode(&m);
F
flame 已提交
1583
  BindInferenceApi(&m);
1584
  BindDataset(&m);
1585 1586 1587
#ifdef PADDLE_WITH_DISTRIBUTE
  BindCommunicator(&m);
#endif
L
Luo Tao 已提交
1588
}
1589
}  // namespace pybind
1590
}  // namespace paddle