pybind.cc 67.6 KB
Newer Older
1
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
2 3 4 5 6

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

7
http://www.apache.org/licenses/LICENSE-2.0
8 9 10 11 12 13

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
L
lgone2000 已提交
14
#include <Python.h>
C
chengduoZH 已提交
15
#include <algorithm>
16
#include <cstdlib>
C
chengduoZH 已提交
17
#include <map>
S
sneaxiy 已提交
18
#include <memory>
C
chengduoZH 已提交
19 20 21
#include <mutex>  // NOLINT // for call_once
#include <string>
#include <unordered_map>
22
#include <unordered_set>
C
chengduoZH 已提交
23 24
#include <utility>
#include <vector>
25

Y
Yi Wang 已提交
26 27 28
#include "paddle/fluid/framework/executor.h"
#include "paddle/fluid/framework/feed_fetch_method.h"
#include "paddle/fluid/framework/framework.pb.h"
S
sneaxiy 已提交
29
#include "paddle/fluid/framework/garbage_collector.h"
30
#include "paddle/fluid/framework/ir/coalesce_grad_tensor_pass.h"
31
#include "paddle/fluid/framework/ir/pass_builder.h"
Y
Yi Wang 已提交
32 33 34
#include "paddle/fluid/framework/lod_rank_table.h"
#include "paddle/fluid/framework/lod_tensor.h"
#include "paddle/fluid/framework/lod_tensor_array.h"
S
sneaxiy 已提交
35
#include "paddle/fluid/framework/op_info.h"
36
#include "paddle/fluid/framework/op_registry.h"
Y
Yu Yang 已提交
37
#include "paddle/fluid/framework/parallel_executor.h"
Y
Yi Wang 已提交
38
#include "paddle/fluid/framework/prune.h"
Y
Refine  
Yu Yang 已提交
39
#include "paddle/fluid/framework/reader.h"
S
sneaxiy 已提交
40
#include "paddle/fluid/framework/scope_pool.h"
Y
Yi Wang 已提交
41
#include "paddle/fluid/framework/selected_rows.h"
X
Xin Pan 已提交
42
#include "paddle/fluid/framework/version.h"
Y
Refine  
Yu Yang 已提交
43
#include "paddle/fluid/memory/allocation/allocator_strategy.h"
D
dzhwinter 已提交
44
#include "paddle/fluid/operators/activation_op.h"
S
sneaxiy 已提交
45
#include "paddle/fluid/operators/py_func_op.h"
S
sneaxiy 已提交
46
#include "paddle/fluid/operators/reader/lod_tensor_blocking_queue.h"
47
#include "paddle/fluid/platform/cpu_helper.h"
Y
Yu Yang 已提交
48
#include "paddle/fluid/platform/cpu_info.h"
Y
Yi Wang 已提交
49
#include "paddle/fluid/platform/enforce.h"
50
#include "paddle/fluid/platform/init.h"
Y
Yi Wang 已提交
51 52 53
#include "paddle/fluid/platform/place.h"
#include "paddle/fluid/platform/profiler.h"
#include "paddle/fluid/pybind/const_value.h"
D
dongdaxiang 已提交
54
#include "paddle/fluid/pybind/data_set_py.h"
Y
Yi Wang 已提交
55
#include "paddle/fluid/pybind/exception.h"
D
dongdaxiang 已提交
56
#include "paddle/fluid/pybind/fleet_wrapper_py.h"
57
#include "paddle/fluid/pybind/imperative.h"
F
flame 已提交
58
#include "paddle/fluid/pybind/inference_api.h"
F
flame 已提交
59
#include "paddle/fluid/pybind/ir.h"
60

W
wopeizl 已提交
61
#ifndef _WIN32
D
dongdaxiang 已提交
62
#include "paddle/fluid/pybind/nccl_wrapper_py.h"
W
wopeizl 已提交
63
#endif
64 65
#include "paddle/fluid/pybind/protobuf.h"
#include "paddle/fluid/pybind/pybind.h"  // NOLINT
S
sneaxiy 已提交
66
#include "paddle/fluid/pybind/reader_py.h"
Y
Yu Yang 已提交
67
#include "paddle/fluid/pybind/recordio.h"
Y
Yi Wang 已提交
68
#include "paddle/fluid/pybind/tensor_py.h"
69
#include "paddle/fluid/string/to_string.h"
70

D
Dong Zhihong 已提交
71
#ifdef PADDLE_WITH_CUDA
P
peizhilin 已提交
72
#ifndef _WIN32
Y
Yi Wang 已提交
73
#include "paddle/fluid/operators/nccl/nccl_gpu_common.h"
P
peizhilin 已提交
74
#endif
Y
Yi Wang 已提交
75 76
#include "paddle/fluid/platform/cuda_profiler.h"
#include "paddle/fluid/platform/gpu_info.h"
D
Dong Zhihong 已提交
77 78
#endif

79 80 81 82
#ifdef PADDLE_WITH_DISTRIBUTE
#include "paddle/fluid/pybind/communicator_py.h"
#endif

M
minqiyang 已提交
83 84
#include "pybind11/stl.h"

85 86 87 88
DEFINE_bool(reader_queue_speed_test_mode, false,
            "If set true, the queue.pop will only get data from queue but not "
            "remove the data from queue for speed testing");

Q
Qiao Longfei 已提交
89 90 91
// disable auto conversion to list in Python
PYBIND11_MAKE_OPAQUE(paddle::framework::LoDTensorArray);

92
namespace paddle {
93
namespace pybind {
94
bool IsCompiledWithCUDA() {
95
#ifndef PADDLE_WITH_CUDA
Q
qijun 已提交
96 97 98 99 100 101
  return false;
#else
  return true;
#endif
}

102 103 104 105 106 107 108 109
bool IsCompiledWithMKLDNN() {
#ifndef PADDLE_WITH_MKLDNN
  return false;
#else
  return true;
#endif
}

110 111 112 113 114 115 116 117
bool IsCompiledWithNGRAPH() {
#ifndef PADDLE_WITH_NGRAPH
  return false;
#else
  return true;
#endif
}

118
bool IsCompiledWithBrpc() {
119
#ifndef PADDLE_WITH_DISTRIBUTE
120 121
  return false;
#endif
122 123 124 125 126 127

#ifdef PADDLE_WITH_GRPC
  return false;
#endif

  return true;
128 129
}

Y
update  
Yancey1989 已提交
130
bool IsCompiledWithDIST() {
Y
Yancey1989 已提交
131
#ifdef PADDLE_WITH_DISTRIBUTE
Y
update  
Yancey1989 已提交
132 133 134 135 136 137
  return true;
#else
  return false;
#endif
}

S
sneaxiy 已提交
138 139 140 141 142 143 144 145 146 147
template <typename PlaceType1, typename PlaceType2>
static inline bool IsSamePlace(const PlaceType1 &p1, const PlaceType2 &p2) {
  return paddle::platform::Place(p1) == paddle::platform::Place(p2);
}

template <typename PlaceType>
static inline int PlaceIndex(const PlaceType &p) {
  return static_cast<int>(paddle::platform::Place(p).which());
}

148 149 150 151 152 153
#ifdef PADDLE_WITH_AVX
PYBIND11_MODULE(core_avx, m) {
#else
PYBIND11_MODULE(core_noavx, m) {
#endif

Y
Yu Yang 已提交
154 155 156
  // Not used, just make sure cpu_info.cc is linked.
  paddle::platform::CpuTotalPhysicalMemory();

Y
Refine  
Yu Yang 已提交
157
  paddle::memory::allocation::UseAllocatorStrategyGFlag();
S
sneaxiy 已提交
158

159
  m.doc() = "C++ core of PaddlePaddle";
160

161 162 163 164
  // using framework in this function. Since it is inside a function, it will
  // not cause namespace pollution.
  using namespace paddle::framework;  // NOLINT

165
  BindException(&m);
Y
Yu Yang 已提交
166

167 168
  m.def("set_num_threads", &platform::SetNumThreads);

S
sneaxiy 已提交
169
  m.def(
S
sneaxiy 已提交
170
      "_append_python_callable_object_and_return_id",
S
sneaxiy 已提交
171 172 173 174
      [](py::object py_obj) -> size_t {
        return paddle::operators::AppendPythonCallableObjectAndReturnId(py_obj);
      });

S
sneaxiy 已提交
175 176 177
  m.def("_get_use_default_grad_op_desc_maker_ops",
        [] { return OpInfoMap::Instance().GetUseDefaultGradOpDescMakerOps(); });

S
sneaxiy 已提交
178 179 180
  // NOTE(zjl): ctest would load environment variables at the beginning even
  // though we have not `import paddle.fluid as fluid`. So we add this API
  // to enable eager deletion mode in unittest.
S
sneaxiy 已提交
181
  m.def("_set_eager_deletion_mode", &paddle::framework::SetEagerDeletionMode);
S
sneaxiy 已提交
182

183
  m.def("_set_fuse_parameter_group_size",
184
        &paddle::framework::ir::SetFuseParameterGroupsSize);
185
  m.def("_set_fuse_parameter_memory_size",
186
        &paddle::framework::ir::SetFuseParameterMemorySize);
187

S
sneaxiy 已提交
188 189 190
  m.add_object("_cleanup",
               py::capsule([]() { ScopePool::Instance().Clear(); }));

191
  BindImperative(&m);
192

193
  py::class_<Tensor>(m, "Tensor", py::buffer_protocol())
194
      .def("__array__", [](Tensor &self) { return TensorToPyArray(self); })
S
sneaxiy 已提交
195 196
      .def("_is_initialized",
           [](const Tensor &self) { return self.IsInitialized(); })
Y
yuyang18 已提交
197
      .def("_get_dims",
198
           [](const Tensor &self) { return vectorize(self.dims()); })
Y
yuyang18 已提交
199
      .def("_set_dims",
Q
qijun 已提交
200
           [](Tensor &self, const std::vector<int64_t> &dim) {
Y
Yu Yang 已提交
201
             self.Resize(make_ddim(dim));
Y
Yu Yang 已提交
202
           })
Y
yuyang18 已提交
203
      .def("_set_layout",
D
dzhwinter 已提交
204 205 206
           [](Tensor &self, const std::string &layout) {
             self.set_layout(StringToDataLayout(layout));
           })
Y
yuyang18 已提交
207
      .def("_alloc_float",
D
dzhwinter 已提交
208
           [](Tensor &self, paddle::platform::CUDAPlace &place) {
Q
qijun 已提交
209
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
210
           })
Y
yuyang18 已提交
211
      .def("_alloc_float",
Y
Yu Yang 已提交
212
           [](Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
213
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
214
           })
Y
yuyang18 已提交
215
      .def("_alloc_int",
Y
Yu Yang 已提交
216
           [](Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
217
             self.mutable_data<int>(place);
Y
Yu Yang 已提交
218
           })
Y
yuyang18 已提交
219
      .def("_alloc_int",
D
dzhwinter 已提交
220
           [](Tensor &self, paddle::platform::CUDAPlace &place) {
Q
qijun 已提交
221
             self.mutable_data<int>(place);
Q
qijun 已提交
222
           })
Y
yuyang18 已提交
223
      .def("_alloc_int",
C
chengduoZH 已提交
224 225 226
           [](Tensor &self, paddle::platform::CUDAPinnedPlace &place) {
             self.mutable_data<int>(place);
           })
Y
yuyang18 已提交
227
      .def("_alloc_float",
C
chengduoZH 已提交
228 229 230
           [](Tensor &self, paddle::platform::CUDAPinnedPlace &place) {
             self.mutable_data<float>(place);
           })
Z
Zeng Jinle 已提交
231
      .def("_clear", &Tensor::clear)
Y
Yu Yang 已提交
232 233
      .def("set", PyCPUTensorSetFromArray<float>)
      .def("set", PyCPUTensorSetFromArray<int>)
234
      .def("set", PyCPUTensorSetFromArray<double>)
235
      .def("set", PyCPUTensorSetFromArray<int64_t>)
Y
Yu Yang 已提交
236
      .def("set", PyCPUTensorSetFromArray<bool>)
237
      .def("set", PyCPUTensorSetFromArray<uint16_t>)
F
fengjiayi 已提交
238
      .def("set", PyCPUTensorSetFromArray<uint8_t>)
Q
qingqing01 已提交
239
      .def("set", PyCPUTensorSetFromArray<int8_t>)
240
#ifdef PADDLE_WITH_CUDA
Y
Yu Yang 已提交
241 242
      .def("set", PyCUDATensorSetFromArray<float>)
      .def("set", PyCUDATensorSetFromArray<int>)
243
      .def("set", PyCUDATensorSetFromArray<double>)
244
      .def("set", PyCUDATensorSetFromArray<int64_t>)
Y
Yu Yang 已提交
245
      .def("set", PyCUDATensorSetFromArray<bool>)
246
      .def("set", PyCUDATensorSetFromArray<uint16_t>)
F
fengjiayi 已提交
247
      .def("set", PyCUDATensorSetFromArray<uint8_t>)
Q
qingqing01 已提交
248
      .def("set", PyCUDATensorSetFromArray<int8_t>)
C
chengduoZH 已提交
249 250 251 252 253 254
      .def("set", PyCUDAPinnedTensorSetFromArray<float>)
      .def("set", PyCUDAPinnedTensorSetFromArray<int>)
      .def("set", PyCUDAPinnedTensorSetFromArray<double>)
      .def("set", PyCUDAPinnedTensorSetFromArray<int64_t>)
      .def("set", PyCUDAPinnedTensorSetFromArray<bool>)
      .def("set", PyCUDAPinnedTensorSetFromArray<uint16_t>)
F
fengjiayi 已提交
255
      .def("set", PyCUDAPinnedTensorSetFromArray<uint8_t>)
Q
qingqing01 已提交
256
      .def("set", PyCUDAPinnedTensorSetFromArray<int8_t>)
Q
qijun 已提交
257
#endif
258
      .def("shape", [](Tensor &self) { return vectorize(self.dims()); })
Y
yuyang18 已提交
259 260 261 262
      .def("_set_float_element", TensorSetElement<float>)
      .def("_get_float_element", TensorGetElement<float>)
      .def("_set_double_element", TensorSetElement<double>)
      .def("_get_double_element", TensorGetElement<double>)
X
xuezhong 已提交
263
      .def("_place", [](Tensor &self) { return self.place(); })
W
wopeizl 已提交
264
      .def("_dtype", [](Tensor &self) { return self.type(); })
265 266 267 268 269 270
      .def("__getitem__", PySliceTensor, py::return_value_policy::reference)
      .def("__str__", [](const Tensor &self) {
        std::stringstream ostr;
        ostr << self;
        return ostr.str();
      });
Y
Yu Yang 已提交
271

X
Xin Pan 已提交
272 273 274 275 276 277 278 279 280
  py::class_<LoDTensor, Tensor>(m, "LoDTensor", R"DOC(
    LoDTensor is a Tensor with optional LoD information.

    np.array(lod_tensor) can convert LoDTensor to numpy array.
    lod_tensor.lod() can retrieve the LoD information.

    LoD is short for Level of Details and is usually used for varied sequence
    length. You can skip the following comment if you don't need optional LoD.

281 282
    For example, a LoDTensor X can look like the example below. It contains
    2 sequences. The first has length 2 and the second has length 3, as
Z
Zeng Jinle 已提交
283
    described by x.lod.
X
Xin Pan 已提交
284

Z
Zeng Jinle 已提交
285 286 287
    The first tensor dimension 5=2+3 is calculated from LoD if it's available.
    It means the total number of sequence element. In X, each element has 2
    columns, hence [5, 2].
X
Xin Pan 已提交
288

Z
Zeng Jinle 已提交
289
    x.lod  = [[2, 3]]
290

Z
Zeng Jinle 已提交
291
    x.data = [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]]
X
Xin Pan 已提交
292

Z
Zeng Jinle 已提交
293
    x.shape = [5, 2]
X
Xin Pan 已提交
294

Z
Zeng Jinle 已提交
295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311
    LoD can have multiple levels (for example, a paragraph can have multiple
    sentences and a sentence can have multiple words). In the following
    LodTensor Y, the lod_level is 2. It means there are 2 sequence, the
    first sequence length is 2 (has 2 sub-sequences), the second one's
    length is 1. The first sequence's 2 sub-sequences have length 2 and 2,
    respectively. And the second sequence's 1 sub-sequence has length 3.

    y.lod = [[2 1], [2 2 3]]

    y.shape = [2+2+3, ...]

    Examples:
        .. code-block:: python

          import paddle.fluid as fluid

          t = fluid.LoDTensor()
X
Xin Pan 已提交
312 313 314 315 316 317 318 319 320 321 322 323

  Note:
      In above description, LoD is length-based. In Paddle internal
      implementation, lod is offset-based. Hence, internally,
      y.lod is represented as [[0, 2, 3], [0, 2, 4, 7]] (length-based
      equivlent would be [[2-0, 3-2], [2-0, 4-2, 7-4]]).

      Sometimes LoD is called recursive_sequence_length to be more
      self-explanatory. In this case, it must be length-based. Due to history
      reasons. when LoD is called lod in public API, it might be offset-based.
      Users should be careful about it.
        )DOC")
324
      .def("__array__", [](Tensor &self) { return TensorToPyArray(self); })
325 326 327 328 329 330 331 332 333 334 335 336 337 338
      .def("__init__",
           [](LoDTensor &instance, const std::vector<std::vector<size_t>>
                                       &recursive_sequence_lengths) {
             LoD new_lod;
             new_lod.reserve(recursive_sequence_lengths.size());
             std::copy(recursive_sequence_lengths.begin(),
                       recursive_sequence_lengths.end(),
                       std::back_inserter(new_lod));
             LoD new_offset_lod = ConvertToOffsetBasedLoD(new_lod);
             PADDLE_ENFORCE(
                 CheckLoD(new_offset_lod, -1),
                 "the provided recursive_sequence_lengths info is invalid");
             new (&instance) LoDTensor(new_offset_lod);
           })
Y
Yu Yang 已提交
339
      .def("__init__", [](LoDTensor &instance) { new (&instance) LoDTensor(); })
G
gongweibao 已提交
340 341 342 343 344
      // We implement offset based LOD in C++ while we use length based with
      // Python API. So we changed set_lod to set_recursive_sequence_lengths to
      // avoid misuse.
      // The discussion is here:
      // https://github.com/PaddlePaddle/Paddle/issues/10855
D
dangqingqing 已提交
345
      .def("set_lod",
346
           [](LoDTensor &self, const std::vector<std::vector<size_t>> &lod) {
347
             // the input lod is offset-based level-of-detail info
Y
Yu Yang 已提交
348
             LoD new_lod;
349 350
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
351 352
             PADDLE_ENFORCE(CheckLoD(new_lod, vectorize(self.dims()).front()),
                            "the provided lod info is invalid");
353
             self.set_lod(new_lod);
S
sneaxiy 已提交
354 355 356 357 358 359
           },
           py::arg("lod"), R"DOC(
           Set LoD of the LoDTensor.

           Args:
               lod (List[List[int]]): the lod to be set.
Z
Zeng Jinle 已提交
360 361 362 363 364 365 366 367 368 369

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_lod([[0, 2, 5]])
S
sneaxiy 已提交
370
           )DOC")
371 372 373 374 375 376 377 378 379 380 381 382 383 384 385
      .def("set_recursive_sequence_lengths",
           [](LoDTensor &self, const std::vector<std::vector<size_t>>
                                   &recursive_sequence_lengths) {
             // the input recursive_sequence_lengths is length-based
             // level-of-detail info
             LoD new_lod;
             new_lod.reserve(recursive_sequence_lengths.size());
             std::copy(recursive_sequence_lengths.begin(),
                       recursive_sequence_lengths.end(),
                       std::back_inserter(new_lod));
             LoD new_offset_lod = ConvertToOffsetBasedLoD(new_lod);
             PADDLE_ENFORCE(
                 CheckLoD(new_offset_lod, vectorize(self.dims()).front()),
                 "the provided recursive_sequence_lengths info is invalid");
             self.set_lod(new_offset_lod);
S
sneaxiy 已提交
386 387 388 389
           },
           py::arg("recursive_sequence_lengths"), R"DOC(
           Set LoD of the LoDTensor according to recursive sequence length.

S
sneaxiy 已提交
390
           For example, if recursive_sequence_lengths=[[2, 3]], meaning that
391 392
           there are two sequences with length 2 and 3 respectively, the
           corresponding lod would be [[0, 2, 2+3]], i.e, [[0, 2, 5]].
S
sneaxiy 已提交
393 394

           Args:
395
                recursive_sequence_lengths (List[List[int]]): sequence lengths.
Z
Zeng Jinle 已提交
396 397 398 399 400 401 402 403 404 405

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_recursive_sequence_lengths([[2, 3]])
S
sneaxiy 已提交
406
           )DOC")
407 408 409 410 411 412 413 414
      .def("lod",
           [](LoDTensor &self) -> std::vector<std::vector<size_t>> {
             // output the offset-based lod info
             LoD lod = self.lod();
             std::vector<std::vector<size_t>> new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             return new_lod;
S
sneaxiy 已提交
415 416 417 418 419 420
           },
           R"DOC(
           Return the LoD of the LoDTensor.

           Returns:
               out (List[List[int]]): the lod of the LoDTensor.
Z
Zeng Jinle 已提交
421 422 423 424 425 426 427 428 429 430 431

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_lod([[0, 2, 5]])
                 print(t.lod()) # [[0, 2, 5]]
S
sneaxiy 已提交
432
           )DOC")
G
gongweibao 已提交
433
      // Set above comments of set_lod.
434 435 436 437 438 439 440 441
      .def("recursive_sequence_lengths",
           [](LoDTensor &self) -> std::vector<std::vector<size_t>> {
             // output the length-based lod info
             LoD lod = ConvertToLengthBasedLoD(self.lod());
             std::vector<std::vector<size_t>> new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             return new_lod;
S
sneaxiy 已提交
442 443 444 445 446
           },
           R"DOC(
           Return the sequence length of the LoDTensor corresponding to LoD.

           Returns:
447
               out (List[List[int]): the sequence lengths.
Z
Zeng Jinle 已提交
448 449 450 451 452 453 454 455 456 457 458

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_recursive_sequence_lengths([[2, 3]])
                 print(t.recursive_sequence_lengths()) # [[2, 3]]
S
sneaxiy 已提交
459 460 461 462 463 464 465 466 467 468 469 470
           )DOC")
      .def("has_valid_recursive_sequence_lengths",
           [](LoDTensor &self) -> bool {
             // Check that the lod info is valid and match the outermost
             // dimension of the LoDTensor data
             return CheckLoD(self.lod(), vectorize(self.dims()).front());
           },
           R"DOC(
           Check whether the lod of the LoDTensor is valid.

           Returns:
               out (bool): whether the lod is valid.
Z
Zeng Jinle 已提交
471 472 473 474 475 476 477 478 479 480 481

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_recursive_sequence_lengths([[2, 3]])
                 print(t.has_valid_recursive_sequence_lengths()) # True
W
wopeizl 已提交
482 483 484 485 486 487 488
           )DOC")
      .def("__getitem__", PySliceTensor, py::return_value_policy::reference,
           R"DOC(
           Slice the original Tensor, and remove the LoD information.

           Returns:
               out (Tensor): new Tensor(NOT LoDTensor).
489 490 491 492 493 494
           )DOC")
      .def("__str__", [](const LoDTensor &self) {
        std::stringstream ostr;
        ostr << self;
        return ostr.str();
      });
D
dangqingqing 已提交
495

Q
qijun 已提交
496 497 498 499 500 501 502 503 504 505 506
  py::class_<SelectedRows>(m, "SelectedRows")
      .def("__init__",
           [](SelectedRows &instance) { new (&instance) SelectedRows(); })
      .def("__init__",
           [](SelectedRows &instance, const std::vector<int64_t> rows,
              const int64_t &height) {
             new (&instance) SelectedRows(rows, height);
           })
      .def("get_tensor",
           [](SelectedRows &self) { return self.mutable_value(); },
           py::return_value_policy::reference)
507 508
      .def("numel",
           [](SelectedRows &self) -> int64_t { return self.value().numel(); })
Q
qijun 已提交
509 510
      .def("set_height", &SelectedRows::set_height)
      .def("height", &SelectedRows::height)
Q
qijun 已提交
511 512 513 514 515 516 517 518 519
      .def("set_rows",
           [](SelectedRows &self, std::vector<int64_t> rows) {
#ifndef PADDLE_WITH_CUDA
             self.set_rows(rows);
#else
        Vector<int64_t> new_rows(rows);
        self.set_rows(new_rows);
#endif
           })
520
      .def("sync_index", [](SelectedRows &instance) { instance.SyncIndex(); })
521
      .def("rows", [](SelectedRows &self) {
522 523 524 525 526
        auto rows = self.rows();
        std::vector<int64_t> new_rows;
        new_rows.reserve(rows.size());
        std::copy(rows.begin(), rows.end(), std::back_inserter(new_rows));
        return new_rows;
527
      });
Q
qijun 已提交
528

529
  py::class_<Variable>(m, "Variable", R"DOC(Variable Class.
530 531 532

All parameter, weight, gradient are variables in Paddle.
)DOC")
S
sneaxiy 已提交
533
      .def(py::init<>())
534
      .def("is_int", [](const Variable &var) { return var.IsType<int>(); })
535
      .def("set_int",
536 537
           [](Variable &var, int val) -> void { *var.GetMutable<int>() = val; })
      .def("get_int", [](const Variable &var) -> int { return var.Get<int>(); })
538 539 540 541 542 543 544
      .def("is_float", [](const Variable &var) { return var.IsType<float>(); })
      .def("set_float",
           [](Variable &var, float val) -> void {
             *var.GetMutable<float>() = val;
           })
      .def("get_float",
           [](const Variable &var) -> float { return var.Get<float>(); })
Y
Yu Yang 已提交
545
      .def("get_tensor",
546 547
           [](Variable &self) -> LoDTensor * {
             return self.GetMutable<LoDTensor>();
D
dangqingqing 已提交
548 549
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
550 551 552
      .def("get_lod_rank_table",
           [](Variable &self) { return self.GetMutable<LoDRankTable>(); },
           py::return_value_policy::reference)
Q
qijun 已提交
553 554 555 556 557
      .def("get_selected_rows",
           [](Variable &self) -> SelectedRows * {
             return self.GetMutable<SelectedRows>();
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
558 559 560
      .def("get_lod_tensor_array",
           [](Variable &self) { return self.GetMutable<LoDTensorArray>(); },
           py::return_value_policy::reference)
P
peizhilin 已提交
561
#if (defined(PADDLE_WITH_CUDA) && !defined(_WIN32))
D
Dong Zhihong 已提交
562 563 564 565 566
      .def("get_communicator",
           [](Variable &self) -> platform::Communicator * {
             return self.GetMutable<platform::Communicator>();
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
567
#endif
Y
Refine  
Yu Yang 已提交
568 569 570 571 572
      .def("get_reader",
           [](Variable &self) -> framework::ReaderHolder * {
             PADDLE_ENFORCE(self.IsType<framework::ReaderHolder>());
             return self.GetMutable<framework::ReaderHolder>();
           },
W
wopeizl 已提交
573
           py::return_value_policy::reference);
574

S
sneaxiy 已提交
575
  BindReader(&m);
Y
Refine  
Yu Yang 已提交
576

S
sneaxiy 已提交
577 578 579 580
  using LoDTensorBlockingQueue =
      ::paddle::operators::reader::LoDTensorBlockingQueue;
  using LoDTensorBlockingQueueHolder =
      ::paddle::operators::reader::LoDTensorBlockingQueueHolder;
S
sneaxiy 已提交
581

S
sneaxiy 已提交
582 583
  py::class_<LoDTensorBlockingQueue, std::shared_ptr<LoDTensorBlockingQueue>>(
      m, "LoDTensorBlockingQueue", "")
S
sneaxiy 已提交
584
      .def("push",
S
sneaxiy 已提交
585
           [](LoDTensorBlockingQueue &self,
S
sneaxiy 已提交
586
              const std::vector<framework::LoDTensor> &lod_tensor_vec) {
S
sneaxiy 已提交
587
             pybind11::gil_scoped_release release;
S
sneaxiy 已提交
588
             return self.Push(lod_tensor_vec);
S
sneaxiy 已提交
589
           })
S
sneaxiy 已提交
590 591 592 593
      .def("size", &LoDTensorBlockingQueue::Size)
      .def("capacity", &LoDTensorBlockingQueue::Cap)
      .def("close", &LoDTensorBlockingQueue::Close)
      .def("is_closed", &LoDTensorBlockingQueue::IsClosed);
S
sneaxiy 已提交
594

S
sneaxiy 已提交
595
  m.def("init_lod_tensor_blocking_queue",
Q
Qiao Longfei 已提交
596 597
        [](Variable &var,
           size_t capacity) -> std::shared_ptr<LoDTensorBlockingQueue> {
Q
Qiao Longfei 已提交
598
          VLOG(1) << "init_lod_tensor_blocking_queue";
Q
Qiao Longfei 已提交
599 600 601 602
          auto *holder = var.GetMutable<LoDTensorBlockingQueueHolder>();
          holder->InitOnce(capacity, FLAGS_reader_queue_speed_test_mode);
          return holder->GetQueue();
        },
S
sneaxiy 已提交
603
        py::return_value_policy::copy);
S
sneaxiy 已提交
604

S
sneaxiy 已提交
605
  py::class_<Scope>(m, "_Scope", R"DOC(
Q
Qiao Longfei 已提交
606 607 608 609 610 611 612 613 614 615 616 617 618
    Scope is an association of a name to Variable. All variables belong to Scope.

    Variables in a parent scope can be retrieved from local scope.

    You need to specify a scope to run a Net, i.e., `exe.Run(&scope)`.
    One net can run in different scopes and update different variable in the
    scope.

    You can create var in a scope and get it from the scope.

    Examples:
        .. code-block:: python

619
          import paddle.fluid as fluid
Q
Qiao Longfei 已提交
620 621 622 623 624 625
          # create tensor from a scope and set value to it.
          param = scope.var('Param').get_tensor()
          param_array = np.full((height, row_numel), 5.0).astype("float32")
          param.set(param_array, place)

        )DOC")
S
sneaxiy 已提交
626 627
      .def("_remove_from_pool",
           [](Scope &self) { ScopePool::Instance().Remove(&self); })
D
dongzhihong 已提交
628
      .def("var",
629
           [](Scope &self, const std::string &name) -> Variable * {
D
dongzhihong 已提交
630
             return self.Var(name);
Y
Yu Yang 已提交
631
           },
S
sneaxiy 已提交
632 633
           py::arg("name"),
           R"DOC(
634
           Find or create variable named :code:`name` in the current scope.
S
sneaxiy 已提交
635

636
           If the variable named :code:`name` does not exist in the
S
sneaxiy 已提交
637
           current scope, the variable would be created. Otherwise,
638
           return the existing variable.
S
sneaxiy 已提交
639 640

           Args:
641 642
               name (str): the variable name.

S
sneaxiy 已提交
643
           Returns:
644
               out (core.Variable): the found or created variable.
S
sneaxiy 已提交
645 646 647 648
           )DOC",
           py::return_value_policy::reference)
      .def("find_var", &Scope::FindVar, py::arg("name"),
           R"DOC(
649
           Find variable named :code:`name` in the current scope or
S
sneaxiy 已提交
650
           its parent scope. Return None if not found.
651

S
sneaxiy 已提交
652 653
           Args:
               name (str): the variable name.
654

S
sneaxiy 已提交
655
           Returns:
656
               out (core.Variable|None): the found variable or None.
S
sneaxiy 已提交
657
           )DOC",
658
           py::return_value_policy::reference)
659
      .def("new_scope", [](Scope &self) -> Scope * { return &self.NewScope(); },
S
sneaxiy 已提交
660 661 662 663 664 665
           R"DOC(
           Create a new sub-scope of the current scope.

           Returns:
               out (core._Scope): the created sub-scope.
           )DOC",
666
           py::return_value_policy::reference)
S
sneaxiy 已提交
667 668 669
      .def("drop_kids", &Scope::DropKids,
           R"DOC(
           Delete all sub-scopes of the current scope.
S
sneaxiy 已提交
670 671
           )DOC")
      .def("_kids", &Scope::kids);
672

S
sneaxiy 已提交
673 674 675 676 677 678
  m.def("Scope",
        []() -> Scope * {
          auto *s = new Scope();
          ScopePool::Instance().Insert(std::unique_ptr<Scope>(s));
          return s;
        },
S
sneaxiy 已提交
679 680
        R"DOC(
        Create a new scope.
681

S
sneaxiy 已提交
682 683 684
        Returns:
            out (core._Scope): the created scope.
        )DOC",
S
sneaxiy 已提交
685 686
        py::return_value_policy::reference);

Y
Yu Yang 已提交
687 688
  //! @note: Be careful! PyBind will return std::string as an unicode, not
  //! Python str. If you want a str object, you should cast them in Python.
Y
Yu Yang 已提交
689 690
  m.def("get_all_op_protos", []() -> std::vector<py::bytes> {
    std::vector<py::bytes> ret_values;
691 692 693 694 695 696 697 698 699 700
    for (auto &iter : OpInfoMap::Instance().map()) {
      auto &info = iter.second;
      if (info.HasOpProtoAndChecker()) {
        std::string str;
        PADDLE_ENFORCE(
            info.Proto().SerializeToString(&str),
            "Serialize OpProto Error. This could be a bug of Paddle.");
        ret_values.emplace_back(str);
      }
    }
Y
Yu Yang 已提交
701 702
    return ret_values;
  });
703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718
  m.def(
      "get_grad_op_desc", [](const OpDesc &op_desc,
                             const std::unordered_set<std::string> &no_grad_set,
                             const std::vector<BlockDesc *> &grad_sub_block) {
        std::unordered_map<std::string, std::string> grad_to_var;
        std::vector<std::unique_ptr<OpDesc>> grad_op_descs =
            framework::OpInfoMap::Instance()
                .Get(op_desc.Type())
                .GradOpMaker()(op_desc, no_grad_set, &grad_to_var,
                               grad_sub_block);
        std::vector<OpDesc *> grad_op_desc_ptrs(grad_op_descs.size());
        std::transform(grad_op_descs.begin(), grad_op_descs.end(),
                       grad_op_desc_ptrs.begin(),
                       [](std::unique_ptr<OpDesc> &p) { return p.release(); });
        return std::make_pair(grad_op_desc_ptrs, grad_to_var);
      });
Y
Yu Yang 已提交
719
  m.def("prune", [](const ProgramDesc &origin,
720
                    const std::vector<std::array<size_t, 2>> &targets) {
Y
Yu Yang 已提交
721
    ProgramDesc prog_with_targets(origin);
722
    for (const auto &t : targets) {
723
      prog_with_targets.MutableBlock(t[0])->Op(t[1])->SetIsTarget(true);
724
    }
725
    proto::ProgramDesc pruned_desc;
726
    Prune(*prog_with_targets.Proto(), &pruned_desc);
Y
Yu Yang 已提交
727
    return new ProgramDesc(pruned_desc);
728
  });
729 730 731 732
  m.def("empty_var_name",
        []() { return std::string(framework::kEmptyVarName); });
  m.def("grad_var_suffix",
        []() { return std::string(framework::kGradVarSuffix); });
733 734 735
  m.def_submodule(
       "var_names",
       "The module will return special predefined variable name in Paddle")
Y
Yi Wang 已提交
736 737
      .def("empty", []() { return kEmptyVarName; })
      .def("temp", []() { return kTempVarName; });
Q
qijun 已提交
738
  // clang-format off
Y
Yu Yang 已提交
739
  py::class_<paddle::platform::DeviceContext>(m, "DeviceContext")
Q
qijun 已提交
740 741
      .def_static("create",
                  [](paddle::platform::CPUPlace& place)
Q
qijun 已提交
742
                      -> paddle::platform::DeviceContext* {
Q
qijun 已提交
743 744 745
                    return new paddle::platform::CPUDeviceContext();
                  })
      .def_static("create",
D
dzhwinter 已提交
746
                  [](paddle::platform::CUDAPlace& place)
Q
qijun 已提交
747
                      -> paddle::platform::DeviceContext* {
748
#ifndef PADDLE_WITH_CUDA
D
dzhwinter 已提交
749
                    PADDLE_THROW("CUDAPlace is not supported in CPU device.");
Q
qijun 已提交
750
#else
Q
qijun 已提交
751
                    return new paddle::platform::CUDADeviceContext(place);
Q
qijun 已提交
752
#endif
C
chengduoZH 已提交
753 754 755 756 757 758 759 760 761 762 763
                  })
          .def_static("create",
                [](paddle::platform::CUDAPinnedPlace& place)
                        -> paddle::platform::DeviceContext* {
#ifndef PADDLE_WITH_CUDA
                  PADDLE_THROW(
                        "CUDAPinnedPlace is not supported in CPU device.");
#else
                  return new paddle::platform::CUDAPinnedDeviceContext(place);
#endif
                });;
D
Dong Zhihong 已提交
764
// clang-format on
P
peizhilin 已提交
765
#if (defined(PADDLE_WITH_CUDA) && !defined(_WIN32))
D
Dong Zhihong 已提交
766 767
  py::class_<platform::Communicator>(m, "Communicator").def(py::init<>());
#endif
768 769 770 771
  py::class_<platform::CUDAPlace>(m, "CUDAPlace", R"DOC(
    CUDAPlace is a descriptor of a device. It represents a GPU, and each CUDAPlace
    has a dev_id to indicate the number of cards represented by the current CUDAPlace.
    The memory of CUDAPlace with different dev_id is not accessible.
L
lujun 已提交
772 773 774 775

    Examples:
        .. code-block:: python

776
          import paddle.fluid as fluid
L
lujun 已提交
777 778
          gpu_place = fluid.CUDAPlace(0)

779
        )DOC")
S
sneaxiy 已提交
780 781 782
      .def("__init__",
           [](platform::CUDAPlace &self, int dev_id) {
#ifdef PADDLE_WITH_CUDA
783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806
             if (UNLIKELY(dev_id < 0)) {
               LOG(ERROR) << string::Sprintf(
                   "Invalid CUDAPlace(%d), device id must be 0 or "
                   "positive integer",
                   dev_id);
               std::exit(-1);
             }

             if (UNLIKELY(dev_id >= platform::GetCUDADeviceCount())) {
               if (platform::GetCUDADeviceCount() == 0) {
                 LOG(ERROR) << "Cannot use GPU because there is no GPU "
                               "detected on your "
                               "machine.";
                 std::exit(-1);
               } else {
                 LOG(ERROR) << string::Sprintf(
                     "Invalid CUDAPlace(%d), must inside [0, %d), because GPU "
                     "number on your machine is %d",
                     dev_id, platform::GetCUDADeviceCount(),
                     platform::GetCUDADeviceCount());
                 std::exit(-1);
               }
             }

S
sneaxiy 已提交
807 808
             new (&self) platform::CUDAPlace(dev_id);
#else
809 810 811 812 813 814 815 816 817
             LOG(ERROR) << string::Sprintf(
                 "Cannot use GPU because you have installed CPU version "
                 "PaddlePaddle.\n"
                 "If you want to use GPU, please try to install GPU version "
                 "PaddlePaddle by: pip install paddlepaddle-gpu\n"
                 "If you only have CPU, please change CUDAPlace(%d) to be "
                 "CPUPlace().\n",
                 dev_id);
             std::exit(-1);
S
sneaxiy 已提交
818 819
#endif
           })
S
sneaxiy 已提交
820 821 822 823 824 825
      .def("_type", &PlaceIndex<platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::Place>)
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::CPUPlace>)
      .def("_equals",
           &IsSamePlace<platform::CUDAPlace, platform::CUDAPinnedPlace>)
D
dzhwinter 已提交
826
      .def("__str__", string::to_string<const platform::CUDAPlace &>);
Q
qijun 已提交
827

828 829 830
  py::class_<paddle::platform::CPUPlace>(m, "CPUPlace", R"DOC(
    CPUPlace is a descriptor of a device. It represents a CPU, and the memory
    CPUPlace can be accessed by CPU.
L
lujun 已提交
831 832 833 834

    Examples:
        .. code-block:: python

835
          import paddle.fluid as fluid
L
lujun 已提交
836 837
          cpu_place = fluid.CPUPlace()

838
        )DOC")
839
      .def(py::init<>())
S
sneaxiy 已提交
840 841 842 843 844 845
      .def("_type", &PlaceIndex<platform::CPUPlace>)
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::Place>)
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::CPUPlace>)
      .def("_equals",
           &IsSamePlace<platform::CPUPlace, platform::CUDAPinnedPlace>)
846
      .def("__str__", string::to_string<const platform::CPUPlace &>);
Y
Yu Yang 已提交
847

848 849 850
  py::class_<paddle::platform::CUDAPinnedPlace>(m, "CUDAPinnedPlace", R"DOC(
    CUDAPinnedPlace is a descriptor of a device. The memory of CUDAPinnedPlace
    can be accessed by GPU and CPU.
L
lujun 已提交
851 852 853 854

    Examples:
        .. code-block:: python

855
          import paddle.fluid as fluid
L
lujun 已提交
856 857
          place = fluid.CUDAPinnedPlace()

858
        )DOC")
S
sneaxiy 已提交
859
      .def("__init__",
S
sneaxiy 已提交
860
           [](platform::CUDAPinnedPlace &self) {
S
sneaxiy 已提交
861 862 863
#ifndef PADDLE_WITH_CUDA
             PADDLE_THROW("Cannot use CUDAPinnedPlace in CPU only version");
#endif
S
sneaxiy 已提交
864
             new (&self) platform::CUDAPinnedPlace();
S
sneaxiy 已提交
865
           })
S
sneaxiy 已提交
866 867 868 869 870 871 872 873
      .def("_type", &PlaceIndex<platform::CUDAPinnedPlace>)
      .def("_equals", &IsSamePlace<platform::CUDAPinnedPlace, platform::Place>)
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::CUDAPlace>)
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::CPUPlace>)
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::CUDAPinnedPlace>)
C
chengduoZH 已提交
874 875
      .def("__str__", string::to_string<const platform::CUDAPinnedPlace &>);

Y
Yu Yang 已提交
876 877
  py::class_<platform::Place>(m, "Place")
      .def(py::init<>())
S
sneaxiy 已提交
878 879 880 881 882
      .def("_type", &PlaceIndex<platform::Place>)
      .def("_equals", &IsSamePlace<platform::Place, platform::Place>)
      .def("_equals", &IsSamePlace<platform::Place, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::Place, platform::CPUPlace>)
      .def("_equals", &IsSamePlace<platform::Place, platform::CUDAPinnedPlace>)
X
xuezhong 已提交
883 884
      .def("is_gpu_place",
           [](platform::Place &self) { return platform::is_gpu_place(self); })
S
sneaxiy 已提交
885 886 887 888 889 890
      .def("is_cpu_place",
           [](platform::Place &self) { return platform::is_cpu_place(self); })
      .def("is_cuda_pinned_place",
           [](platform::Place &self) {
             return platform::is_cuda_pinned_place(self);
           })
X
xuezhong 已提交
891 892 893 894
      .def("gpu_device_id",
           [](platform::Place &self) {
             return boost::get<platform::CUDAPlace>(self).device;
           })
S
sneaxiy 已提交
895 896
      .def("set_place", [](platform::Place &self,
                           const platform::Place &other) { self = other; })
Y
Yu Yang 已提交
897 898 899 900 901
      .def("set_place",
           [](platform::Place &self, const platform::CPUPlace &cpu_place) {
             self = cpu_place;
           })
      .def("set_place",
D
dzhwinter 已提交
902
           [](platform::Place &self, const platform::CUDAPlace &gpu_place) {
Y
Yu Yang 已提交
903
             self = gpu_place;
C
chengduoZH 已提交
904 905
           })
      .def("set_place", [](platform::Place &self,
C
chengduoZH 已提交
906 907
                           const platform::CUDAPinnedPlace &cuda_pinned_place) {
        self = cuda_pinned_place;
C
chengduoZH 已提交
908
      });
Y
Yu Yang 已提交
909

Y
Yu Yang 已提交
910 911 912
  py::class_<OperatorBase>(m, "Operator")
      .def_static("create",
                  [](py::bytes protobin) {
913
                    proto::OpDesc desc;
Y
Yu Yang 已提交
914 915 916 917 918
                    PADDLE_ENFORCE(desc.ParsePartialFromString(protobin),
                                   "Cannot parse user input to OpDesc");
                    PADDLE_ENFORCE(desc.IsInitialized(),
                                   "User OpDesc is not initialized, reason %s",
                                   desc.InitializationErrorString());
919
                    return OpRegistry::CreateOp(desc);
Y
Yu Yang 已提交
920
                  })
921
      .def("run",
922
           [](OperatorBase &self, const Scope &scope,
D
dzhwinter 已提交
923 924 925
              const platform::CPUPlace &place) { self.Run(scope, place); })
      .def("run",
           [](OperatorBase &self, const Scope &scope,
D
dzhwinter 已提交
926
              const platform::CUDAPlace &place) { self.Run(scope, place); })
C
chengduoZH 已提交
927 928 929 930 931
      .def("run",
           [](OperatorBase &self, const Scope &scope,
              const platform::CUDAPinnedPlace &place) {
             self.Run(scope, place);
           })
Y
Yu Yang 已提交
932 933 934 935 936 937 938
      .def("type",
           [](const OperatorBase &op) -> std::string { return op.Type(); })
      .def("outputs",
           [](const OperatorBase &op)
               -> std::map<std::string, std::vector<std::string>> {
                 return op.Outputs();
               })
Q
qijun 已提交
939 940
      .def("output_vars",
           [](const OperatorBase &op) { return op.OutputVars(true); })
Y
Yu Yang 已提交
941
      .def("inputs", [](const OperatorBase &op) { return op.Inputs(); })
Q
qijun 已提交
942
      .def("input_vars", [](const OperatorBase &op) { return op.InputVars(); })
Y
Yu Yang 已提交
943 944 945 946
      .def("__str__", &OperatorBase::DebugString)
      .def("no_intermediate_outputs",
           [](const OperatorBase &op) { return op.OutputVars(false); })
      .def("support_gpu", &OperatorBase::SupportGPU);
Y
Yu Yang 已提交
947

948 949 950
  py::class_<framework::ExecutorPrepareContext>(m, "ExecutorPrepareContext")
      .def(py::init<const ProgramDesc &, size_t>());

F
fengjiayi 已提交
951
  py::class_<framework::Executor>(m, "Executor")
D
dzhwinter 已提交
952
      .def(py::init<const platform::Place &>())
Y
Yancey1989 已提交
953
      .def("close", &Executor::Close)
954 955 956 957 958 959 960 961 962 963 964 965 966 967
      .def("run_from_dataset", &Executor::RunFromDataset,
           py::call_guard<py::gil_scoped_release>())
      .def("run_prepared_ctx",
           [](Executor &self, ExecutorPrepareContext *ctx, Scope *scope,
              std::map<std::string, const LoDTensor *> *feed_targets,
              std::map<std::string, LoDTensor *> *fetch_targets,
              bool create_local_scope = true, bool create_vars = true,
              const std::string &feed_holder_name = "feed",
              const std::string &fetch_holder_name = "fetch") {
             pybind11::gil_scoped_release release;
             self.RunPreparedContext(ctx, scope, feed_targets, fetch_targets,
                                     create_local_scope, create_vars,
                                     feed_holder_name, fetch_holder_name);
           })
G
guru4elephant 已提交
968 969 970 971 972 973 974 975
      .def("run_cached_prepared_ctx",
           [](Executor &self, ExecutorPrepareContext *ctx, Scope *scope,
              bool create_local_scope = true, bool create_vars = true,
              bool keep_kids = false) {
             pybind11::gil_scoped_release release;
             self.RunPreparedContext(ctx, scope, create_local_scope,
                                     create_vars, keep_kids);
           })
976 977
      .def("prepare_ctx_cache", &Executor::PrepareCtxCache,
           py::call_guard<py::gil_scoped_release>())
978 979
      .def("create_variables", &Executor::CreateVariables,
           py::call_guard<py::gil_scoped_release>())
S
sneaxiy 已提交
980
      .def("run", [](Executor &self, const ProgramDesc &prog, Scope *scope,
S
sneaxiy 已提交
981 982
                     int block_id, bool create_local_scope, bool create_vars,
                     const std::vector<std::string> &fetch_vars) {
S
sneaxiy 已提交
983
        pybind11::gil_scoped_release release;
S
sneaxiy 已提交
984 985
        self.Run(prog, scope, block_id, create_local_scope, create_vars,
                 fetch_vars);
S
sneaxiy 已提交
986
      });
S
sneaxiy 已提交
987

D
dzhwinter 已提交
988
  m.def("init_gflags", framework::InitGflags);
Y
Yang Yu 已提交
989
  m.def("init_glog", framework::InitGLOG);
990
  m.def("init_dgc", framework::InitDGC);
X
Xin Pan 已提交
991 992
  m.def("init_devices",
        [](bool init_p2p) { framework::InitDevices(init_p2p); });
993

994
  m.def("is_compiled_with_ngraph", IsCompiledWithNGRAPH);
995
  m.def("is_compiled_with_cuda", IsCompiledWithCUDA);
996
  m.def("is_compiled_with_mkldnn", IsCompiledWithMKLDNN);
997
  m.def("is_compiled_with_brpc", IsCompiledWithBrpc);
Y
update  
Yancey1989 已提交
998
  m.def("is_compiled_with_dist", IsCompiledWithDIST);
999 1000 1001 1002 1003 1004
#ifdef PADDLE_WITH_CUDA
  m.def("is_float16_supported", [](const platform::CUDAPlace &place) -> bool {
    // Only GPUs with Compute Capability >= 53 support float16
    return platform::GetCUDAComputeCapability(place.device) >= 53;
  });
#endif
1005

1006
  m.def("set_feed_variable", framework::SetFeedVariable);
Q
qijun 已提交
1007
  m.def("get_fetch_variable", framework::GetFetchVariable);
1008
  m.def("get_variable_tensor", framework::GetVariableTensor);
Q
qijun 已提交
1009

X
Xin Pan 已提交
1010 1011
  m.def("_is_program_version_supported", IsProgramVersionSupported);

1012 1013 1014 1015 1016
  BindProgramDesc(&m);
  BindBlockDesc(&m);
  BindVarDsec(&m);
  BindOpDesc(&m);
  BindConstValue(&m);
Y
Yu Yang 已提交
1017

Y
Yu Yang 已提交
1018 1019 1020 1021 1022 1023 1024 1025 1026
  py::class_<framework::LoDRankTable>(m, "LodRankTable")
      .def("items", [](framework::LoDRankTable &table) {
        std::vector<std::pair<size_t, size_t>> res;
        for (auto &item : table.items()) {
          res.push_back({item.index, item.length});
        }
        return res;
      });

Z
Zeng Jinle 已提交
1027 1028 1029 1030 1031
  py::class_<LoDTensorArray>(m, "LoDTensorArray", R"DOC(
    Array of LoDTensor.

    Examples:
        .. code-block:: python
1032

Z
Zeng Jinle 已提交
1033 1034 1035 1036
          import paddle.fluid as fluid

          arr = fluid.LoDTensorArray()
)DOC")
S
sneaxiy 已提交
1037 1038
      .def("__init__",
           [](LoDTensorArray &instance) { new (&instance) LoDTensorArray(); })
Y
Yu Yang 已提交
1039 1040 1041 1042 1043 1044 1045 1046 1047 1048
      .def("__getitem__",
           [](LoDTensorArray &self, size_t i) { return &self.at(i); },
           py::return_value_policy::reference)
      .def("__len__", [](LoDTensorArray &self) { return self.size(); })
      .def("__setitem__",
           [](LoDTensorArray &self, size_t i, const LoDTensor &t) {
             PADDLE_ENFORCE_LT(i, self.size());
             self[i].ShareDataWith(t);
             self[i].set_lod(t.lod());
           })
S
sneaxiy 已提交
1049 1050 1051 1052 1053 1054
      .def("append",
           [](LoDTensorArray &self, const LoDTensor &t) {
             self.emplace_back();
             self.back().ShareDataWith(t);
             self.back().set_lod(t.lod());
           },
Z
Zeng Jinle 已提交
1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068
           py::arg("tensor"), R"DOC(
             Append a LoDensor to LoDTensorArray.

             Examples:
                 .. code-block:: python

                   import paddle.fluid as fluid
                   import numpy as np

                   arr = fluid.LoDTensorArray()
                   t = fluid.LoDTensor()
                   t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                   arr.append(t)
           )DOC");
Y
Yu Yang 已提交
1069

D
dzhwinter 已提交
1070 1071 1072
  m.def("IsInplace",
        [](std::string op) -> bool { return operators::IsInplace(op); });

Y
Yu Yang 已提交
1073
  m.def("op_support_gpu", OpSupportGPU);
D
Dong Zhihong 已提交
1074
#ifdef PADDLE_WITH_CUDA
D
Dong Zhihong 已提交
1075
  m.def("get_cuda_device_count", platform::GetCUDADeviceCount);
D
dangqingqing 已提交
1076

P
peizhilin 已提交
1077
#ifndef _WIN32
D
dangqingqing 已提交
1078 1079 1080
  m.def("nvprof_init", platform::CudaProfilerInit);
  m.def("nvprof_start", platform::CudaProfilerStart);
  m.def("nvprof_stop", platform::CudaProfilerStop);
D
Dong Zhihong 已提交
1081
#endif
P
peizhilin 已提交
1082
#endif
Y
Yu Yang 已提交
1083

1084 1085 1086 1087
  py::enum_<platform::ProfilerState>(m, "ProfilerState", py::arithmetic())
      .value("kDisabled", platform::ProfilerState::kDisabled)
      .value("kCPU", platform::ProfilerState::kCPU)
      .value("kCUDA", platform::ProfilerState::kCUDA)
1088
      .value("kAll", platform::ProfilerState::kAll)
1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101
      .export_values();

  py::enum_<platform::EventSortingKey>(m, "EventSortingKey", py::arithmetic())
      .value("kDefault", platform::EventSortingKey::kDefault)
      .value("kCalls", platform::EventSortingKey::kCalls)
      .value("kTotal", platform::EventSortingKey::kTotal)
      .value("kMin", platform::EventSortingKey::kMin)
      .value("kMax", platform::EventSortingKey::kMax)
      .value("kAve", platform::EventSortingKey::kAve)
      .export_values();

  m.def("enable_profiler", platform::EnableProfiler);
  m.def("disable_profiler", platform::DisableProfiler);
X
Xin Pan 已提交
1102
  m.def("is_profiler_enabled", platform::IsProfileEnabled);
1103
  m.def("reset_profiler", platform::ResetProfiler);
1104
  m.def("get_pass", [](const std::string &pass_type) {
W
WangZhen 已提交
1105 1106 1107
    auto pass = framework::ir::PassRegistry::Instance().Get(pass_type);
    return std::shared_ptr<framework::ir::Pass>(std::move(pass));
  });
Y
Yu Yang 已提交
1108

1109 1110
  py::class_<ir::Pass, std::shared_ptr<ir::Pass>> pass(m, "Pass");
  pass.def(py::init())
W
WangZhen 已提交
1111
      .def("has", &ir::Pass::Has)
1112 1113 1114
      .def("set_not_owned",
           [](ir::Pass &self, const std::string &attr_name, ProgramDesc &attr) {
             self.SetNotOwned<ProgramDesc>(attr_name, &attr);
W
WangZhen 已提交
1115
           })
1116
      .def(
1117
          "set",
1118 1119 1120
          [](ir::Pass &self, const std::string &name, const std::string &attr) {
            self.Set<std::string>(name, new std::string(attr));
          })
1121 1122
      .def("set", [](ir::Pass &self, const std::string &name,
                     int val) { self.Set<const int>(name, new int(val)); })
F
flame 已提交
1123 1124
      .def("type", &ir::Pass::Type)
      .def("apply", [](ir::Pass &self, std::shared_ptr<ir::Graph> graph) {
1125
        self.Apply(graph.get());
F
flame 已提交
1126
      });
1127

X
fix  
Xin Pan 已提交
1128 1129
  py::class_<ir::PassBuilder, std::shared_ptr<ir::PassBuilder>> pb(
      m, "PassBuilder");
1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143
  pb.def(py::init())
      .def("append_pass",
           [](ir::PassBuilder &self,
              const std::string &pass_type) -> std::shared_ptr<ir::Pass> {
             return self.AppendPass(pass_type);
           })
      .def("all_passes", [](ir::PassBuilder &self) { return self.AllPasses(); })
      .def("insert_pass",
           [](ir::PassBuilder &self, size_t idx, const std::string &pass_type) {
             return self.InsertPass(idx, pass_type);
           })
      .def("remove_pass",
           [](ir::PassBuilder &self, size_t idx) { self.RemovePass(idx); });

Y
yuyang18 已提交
1144
  // -- python binds for parallel executor.
X
Xin Pan 已提交
1145

Y
yuyang18 已提交
1146
  py::class_<ParallelExecutor> pe(m, "ParallelExecutor");
C
chengduo 已提交
1147 1148 1149 1150
  py::class_<ExecutionStrategy> exec_strategy(pe, "ExecutionStrategy", R"DOC(
    ExecutionStrategy allows the user to more preciously control how to run
    the program in ParallelExecutor by setting the property.

C
chengduo 已提交
1151 1152 1153
    Examples:
        .. code-block:: python

1154
          import paddle.fluid as fluid
1155 1156 1157 1158 1159 1160 1161 1162 1163 1164
          x = fluid.layers.data(name='x', shape=[13], dtype='float32')
          y = fluid.layers.data(name='y', shape=[1], dtype='float32')
          y_predict = fluid.layers.fc(input=x, size=1, act=None)

          cost = fluid.layers.square_error_cost(input=y_predict, label=y)
          avg_loss = fluid.layers.mean(cost)

          sgd_optimizer = fluid.optimizer.SGD(learning_rate=0.001)
          sgd_optimizer.minimize(avg_loss)

C
chengduo 已提交
1165 1166 1167
          exec_strategy = fluid.ExecutionStrategy()
          exec_strategy.num_threads = 4

1168 1169
          train_exe = fluid.ParallelExecutor(use_cuda=False,
                                             loss_name=avg_loss.name,
C
chengduo 已提交
1170 1171
                                             exec_strategy=exec_strategy)

C
chengduo 已提交
1172 1173
        )DOC");

Y
yuyang18 已提交
1174
  exec_strategy.def(py::init())
Y
yuyang18 已提交
1175 1176 1177 1178 1179
      .def_property(
          "num_threads",
          [](const ExecutionStrategy &self) { return self.num_threads_; },
          [](ExecutionStrategy &self, size_t num_threads) {
            self.num_threads_ = num_threads;
C
chengduo 已提交
1180 1181 1182 1183 1184 1185 1186 1187 1188 1189
          },
          R"DOC(The type is INT, num_threads represents the size of thread pool that
            used to run the operators of the current program in ParallelExecutor.
            If :math:`num\_threads=1`, all the operators will execute one by one,
            but the order maybe difference between iterations.
            If it is not set, it will be set in ParallelExecutor according to the
            device type and device count, for GPU, :math:`num\_threads=device\_count*4`, for CPU,
            :math:`num\_threads=CPU\_NUM*4`, the explanation of:math:`CPU\_NUM` is in ParallelExecutor.
            if it is not set, ParallelExecutor will get the cpu count by calling
            `multiprocessing.cpu_count()`. Default 0.)DOC")
Y
yuyang18 已提交
1190
      .def_property(
1191 1192 1193 1194
          "use_cuda",
          [](const ExecutionStrategy &self) { return self.use_cuda_; },
          [](ExecutionStrategy &self, bool use_cuda) {
            self.use_cuda_ = use_cuda;
C
chengduo 已提交
1195 1196 1197 1198
          })  // FIXME(chengduo): Doesn't add doc for 'use_cuda', use_cuda may
      // make user confuse, because ParallelExecutor has a parameter named
      // 'use_cuda' too, in current implementation, ParallelExecutor's
      // 'use_cuda' will rewrite ExecutionStrategy's 'use_cuda'.
Y
yuyang18 已提交
1199 1200 1201 1202 1203
      .def_property(
          "allow_op_delay",
          [](const ExecutionStrategy &self) { return self.allow_op_delay_; },
          [](ExecutionStrategy &self, bool allow_op_delay) {
            self.allow_op_delay_ = allow_op_delay;
C
chengduo 已提交
1204 1205 1206
          },
          R"DOC(The type is BOOL, allow_op_delay represents whether to delay the
                communication operators to run, it may make the execution faster.
1207 1208
                Note that this option is invalid now, and it will be removed in
                next version. Default False.)DOC")
Y
yuyang18 已提交
1209 1210 1211 1212 1213 1214 1215
      .def_property(
          "num_iteration_per_drop_scope",
          [](const ExecutionStrategy &self) {
            return self.num_iteration_per_drop_scope_;
          },
          [](ExecutionStrategy &self, size_t num_iteration_per_drop_scope) {
            self.num_iteration_per_drop_scope_ = num_iteration_per_drop_scope;
C
chengduo 已提交
1216 1217 1218 1219
          },
          R"DOC(The type is INT, num_iteration_per_drop_scope indicates how
                many iterations to clean up the temp variables which
                is generated during execution. It may make the execution faster,
1220 1221
                because the temp variable's shape maybe the same between two iterations.
                Default 1.
C
chengduo 已提交
1222 1223 1224 1225 1226 1227

                NOTES:
                    1. If you fetch data when calling the 'run', the ParallelExecutor
                       will clean up the temp variables at the end of the current iteration.
                    2. In some NLP model, it may cause the GPU memory is insufficient,
                       in this case, you should reduce `num_iteration_per_drop_scope`.
1228
              )DOC")
Q
Qiao Longfei 已提交
1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239
      .def_property(
          "num_iteration_per_run",
          [](const ExecutionStrategy &self) {
            return self.num_iteration_per_run_;
          },
          [](ExecutionStrategy &self, size_t num_iteration_per_run) {
            self.num_iteration_per_run_ = num_iteration_per_run;
          },
          R"DOC(This config that how many iteration the executor will run when
                user call pe.run() in python
              )DOC")
1240 1241 1242 1243 1244
      .def_property("_dry_run",
                    [](const ExecutionStrategy &self) { return self.dry_run_; },
                    [](ExecutionStrategy &self, bool dry_run) {
                      self.dry_run_ = dry_run;
                    });
C
chengduo 已提交
1245

Y
yuyang18 已提交
1246
  exec_strategy.def_property(
Y
yuyang18 已提交
1247 1248 1249 1250 1251 1252 1253
      "use_experimental_executor",
      [](const ExecutionStrategy &self) {
        return self.type_ == ExecutionStrategy::kExperimental;
      },
      [](ExecutionStrategy &self, bool experimental) {
        self.type_ = experimental ? ExecutionStrategy::kExperimental
                                  : ExecutionStrategy::kDefault;
Y
yuyang18 已提交
1254 1255
      });

C
chengduo 已提交
1256 1257 1258 1259
  py::class_<BuildStrategy> build_strategy(pe, "BuildStrategy", R"DOC(
    BuildStrategy allows the user to more preciously control how to
    build the SSA Graph in ParallelExecutor by setting the property.

C
chengduo 已提交
1260 1261 1262
    Examples:
        .. code-block:: python

F
flame 已提交
1263 1264 1265
            import paddle.fluid as fluid
            build_strategy = fluid.BuildStrategy()
            build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce
C
chengduo 已提交
1266
)DOC");
Y
yuyang18 已提交
1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282

  py::enum_<BuildStrategy::ReduceStrategy>(build_strategy, "ReduceStrategy")
      .value("Reduce", BuildStrategy::ReduceStrategy::kReduce)
      .value("AllReduce", BuildStrategy::ReduceStrategy::kAllReduce);
  py::enum_<BuildStrategy::GradientScaleStrategy>(build_strategy,
                                                  "GradientScaleStrategy")
      .value("CoeffNumDevice",
             BuildStrategy::GradientScaleStrategy::kCoeffNumDevice)
      .value("One", BuildStrategy::GradientScaleStrategy::kOne)
      .value("Customized", BuildStrategy::GradientScaleStrategy::kCustomized);

  build_strategy.def(py::init())
      .def_property(
          "reduce_strategy",
          [](const BuildStrategy &self) { return self.reduce_; },
          [](BuildStrategy &self, BuildStrategy::ReduceStrategy strategy) {
X
Xin Pan 已提交
1283
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
Y
yuyang18 已提交
1284
            self.reduce_ = strategy;
C
chengduo 已提交
1285
          },
C
chengduo 已提交
1286 1287 1288 1289 1290 1291 1292
          R"DOC(The type is fluid.BuildStrategy.ReduceStrategy, there are two reduce
                strategies in ParallelExecutor, AllReduce and Reduce. If you want
                that all the parameters' optimization are done on all devices independently,
                you should choose AllReduce; if you choose Reduce, all the parameters'
                optimization will be evenly distributed to different devices, and then
                broadcast the optimized parameter to other devices.
                Default 'AllReduce'.
F
flame 已提交
1293 1294 1295 1296 1297 1298 1299 1300

                Examples:
                    .. code-block:: python

                        import paddle.fluid as fluid
                        build_strategy = fluid.BuildStrategy()
                        build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce
                  )DOC")
Y
yuyang18 已提交
1301 1302 1303 1304 1305
      .def_property(
          "gradient_scale_strategy",
          [](const BuildStrategy &self) { return self.gradient_scale_; },
          [](BuildStrategy &self,
             BuildStrategy::GradientScaleStrategy strategy) {
C
chengduo 已提交
1306
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finalized.");
Y
yuyang18 已提交
1307
            self.gradient_scale_ = strategy;
C
chengduo 已提交
1308
          },
C
chengduo 已提交
1309 1310 1311 1312 1313
          R"DOC(The type is fluid.BuildStrategy.GradientScaleStrategy, there are three
                ways of defining :math:`loss@grad` in ParallelExecutor, CoeffNumDevice,
                One and Customized. By default, ParallelExecutor sets the :math:`loss@grad`
                according to the number of devices. If you want to customize :math:`loss@grad`,
                you can choose Customized. Default 'CoeffNumDevice'.
F
flame 已提交
1314 1315 1316 1317 1318

                Examples:
                    .. code-block:: python

                        import paddle.fluid as fluid
C
chengduo 已提交
1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346
                        import paddle.fluid.compiler as compiler
                        import numpy
                        import os

                        use_cuda = True
                        place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
                        exe = fluid.Executor(place)

                        # NOTE: If you use CPU to run the program, you need
                        # to specify the CPU_NUM, otherwise, fluid will use
                        # all the number of the logic core as the CPU_NUM,
                        # in that case, the batch size of the input should be
                        # greater than CPU_NUM, if not, the process will be
                        # failed by an exception.
                        if not use_cuda:
                            os.environ['CPU_NUM'] = str(2)
                            places = fluid.cpu_places()
                        else:
                            places = places = fluid.cuda_places()

                        data = fluid.layers.data(name='X', shape=[1], dtype='float32')
                        hidden = fluid.layers.fc(input=data, size=10)
                        loss = fluid.layers.mean(hidden)
                        fluid.optimizer.SGD(learning_rate=0.01).minimize(loss)

                        fluid.default_startup_program().random_seed=1
                        exe.run(fluid.default_startup_program())

F
flame 已提交
1347
                        build_strategy = fluid.BuildStrategy()
C
chengduo 已提交
1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361
                        build_strategy.gradient_scale_strategy = \
                                 fluid.BuildStrategy.GradientScaleStrategy.Customized
                        compiled_prog = compiler.CompiledProgram(
                                 fluid.default_main_program()).with_data_parallel(
                                          loss_name=loss.name, build_strategy=build_strategy,
                                          places = places)

                        dev_count =  len(places)
                        x = numpy.random.random(size=(10, 1)).astype('float32')
                        loss_grad = numpy.ones((dev_count)).astype("float32") * 0.01
                        loss_grad_name = loss.name+"@GRAD"
                        loss_data = exe.run(compiled_prog,
                                             feed={"X": x, loss_grad_name : loss_grad},
                                             fetch_list=[loss.name, loss_grad_name])
F
flame 已提交
1362
                   )DOC")
Y
yuyang18 已提交
1363 1364 1365 1366
      .def_property(
          "debug_graphviz_path",
          [](const BuildStrategy &self) { return self.debug_graphviz_path_; },
          [](BuildStrategy &self, const std::string &path) {
X
Xin Pan 已提交
1367
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
Y
yuyang18 已提交
1368
            self.debug_graphviz_path_ = path;
C
chengduo 已提交
1369
          },
C
chengduo 已提交
1370
          R"DOC(The type is STR, debug_graphviz_path indicates the path that
F
flame 已提交
1371 1372 1373 1374 1375 1376 1377 1378
                writing the SSA Graph to file in the form of graphviz.
                It is useful for debugging. Default ""

                Examples:
                    .. code-block:: python

                        import paddle.fluid as fluid
                        build_strategy = fluid.BuildStrategy()
C
chengduo 已提交
1379 1380
                        build_strategy.debug_graphviz_path = "./graph"

F
flame 已提交
1381
                    )DOC")
S
sneaxiy 已提交
1382 1383 1384 1385 1386 1387
      .def_property(
          "enable_sequential_execution",
          [](const BuildStrategy &self) {
            return self.enable_sequential_execution_;
          },
          [](BuildStrategy &self, bool b) {
X
Xin Pan 已提交
1388
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
S
sneaxiy 已提交
1389 1390
            self.enable_sequential_execution_ = b;
          },
C
chengduo 已提交
1391 1392
          R"DOC(The type is BOOL. If set True, the execution order of ops would
                be the same as what is in the program. Default False.
F
flame 已提交
1393 1394 1395 1396 1397 1398 1399 1400

                Examples:
                    .. code-block:: python

                        import paddle.fluid as fluid
                        build_strategy = fluid.BuildStrategy()
                        build_strategy.enable_sequential_execution = True
          )DOC")
S
sneaxiy 已提交
1401 1402 1403 1404 1405 1406
      .def_property(
          "remove_unnecessary_lock",
          [](const BuildStrategy &self) {
            return self.remove_unnecessary_lock_;
          },
          [](BuildStrategy &self, bool b) {
X
Xin Pan 已提交
1407
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
S
sneaxiy 已提交
1408 1409
            self.remove_unnecessary_lock_ = b;
          },
C
chengduo 已提交
1410 1411
          R"DOC(The type is BOOL. If set True, some locks in GPU ops would be
                released and ParallelExecutor would run faster. Default True.
F
flame 已提交
1412 1413 1414 1415 1416 1417 1418 1419

                Examples:
                    .. code-block:: python

                        import paddle.fluid as fluid
                        build_strategy = fluid.BuildStrategy()
                        build_strategy.remove_unnecessary_lock = True
          )DOC")
1420 1421 1422 1423
      .def_property(
          "num_trainers",
          [](const BuildStrategy &self) { return self.num_trainers_; },
          [](BuildStrategy &self, int num_trainers) {
1424 1425 1426
#ifdef WIN32
            PADDLE_THROW("Windows has NO support to distribute mode.");
#endif
1427 1428
            self.num_trainers_ = num_trainers;
          })
1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440
      .def_property(
          "trainers_endpoints",
          [](const BuildStrategy &self) { return self.trainers_endpoints_; },
          [](BuildStrategy &self,
             const std::vector<std::string> &trainers_endpoints) {
            self.trainers_endpoints_ = trainers_endpoints;
          })
      .def_property("trainer_id",
                    [](const BuildStrategy &self) { return self.trainer_id_; },
                    [](BuildStrategy &self, int trainer_id) {
                      self.trainer_id_ = trainer_id;
                    })
1441 1442 1443 1444 1445 1446
      .def_property(
          "nccl_comm_num",
          [](const BuildStrategy &self) { return self.nccl_comm_num_; },
          [](BuildStrategy &self, int nccl_comm_num) {
            self.nccl_comm_num_ = nccl_comm_num;
          })
1447
      .def_property("use_hierarchical_allreduce",
1448 1449 1450 1451 1452 1453
                    [](const BuildStrategy &self) {
                      return self.use_hierarchical_allreduce_;
                    },
                    [](BuildStrategy &self, bool use) {
                      self.use_hierarchical_allreduce_ = use;
                    })
1454
      .def_property("hierarchical_allreduce_inter_nranks",
1455 1456 1457 1458 1459 1460 1461
                    [](const BuildStrategy &self) {
                      return self.hierarchical_allreduce_inter_nranks_;
                    },
                    [](BuildStrategy &self, int nranks) {
                      self.hierarchical_allreduce_inter_nranks_ = nranks;
                    })

C
chengduo 已提交
1462 1463 1464 1465 1466 1467
      .def_property(
          "fuse_elewise_add_act_ops",
          [](const BuildStrategy &self) {
            return self.fuse_elewise_add_act_ops_;
          },
          [](BuildStrategy &self, bool b) {
X
Xin Pan 已提交
1468
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
C
chengduo 已提交
1469 1470 1471
            self.fuse_elewise_add_act_ops_ = b;
          },
          R"DOC(The type is BOOL, fuse_elewise_add_act_ops indicate whether
F
flame 已提交
1472 1473 1474 1475 1476 1477 1478 1479 1480 1481
                to fuse elementwise_add_op and activation_op,
                it may make the execution faster. Default False

                Examples:
                    .. code-block:: python

                        import paddle.fluid as fluid
                        build_strategy = fluid.BuildStrategy()
                        build_strategy.fuse_elewise_add_act_ops = True
                     )DOC")
1482 1483 1484 1485 1486 1487 1488 1489 1490 1491
      .def_property(
          "fuse_relu_depthwise_conv",
          [](const BuildStrategy &self) {
            return self.fuse_relu_depthwise_conv_;
          },
          [](BuildStrategy &self, bool b) {
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
            self.fuse_relu_depthwise_conv_ = b;
          },
          R"DOC(The type is BOOL, fuse_relu_depthwise_conv indicate whether
F
flame 已提交
1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503
                to fuse relu and depthwise_conv2d,
                it will save GPU memory and may make the execution faster.
                This options is only available in GPU devices.
                Default False.

                Examples:
                    .. code-block:: python

                        import paddle.fluid as fluid
                        build_strategy = fluid.BuildStrategy()
                        build_strategy.fuse_relu_depthwise_conv = True
          )DOC")
1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516
      .def_property(
          "fuse_broadcast_ops",
          [](const BuildStrategy &self) { return self.fuse_broadcast_ops_; },
          [](BuildStrategy &self, bool b) {
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
            self.fuse_broadcast_ops_ = b;
          },
          R"DOC(The type is BOOL, fuse_broadcast_op indicates whether
                      to fuse the broadcast ops. Note that, in Reduce mode,
                      fusing broadcast ops may make the program faster. Because
                      fusing broadcast OP equals delaying the execution of all
                      broadcast Ops, in this case, all nccl streams are used only
                      for NCCLReduce operations for a period of time. Default False.)DOC")
C
chengduo 已提交
1517 1518 1519 1520 1521 1522 1523 1524 1525
      .def_property("fuse_all_optimizer_ops",
                    [](const BuildStrategy &self) {
                      return self.fuse_all_optimizer_ops_;
                    },
                    [](BuildStrategy &self, bool b) {
                      PADDLE_ENFORCE(!self.IsFinalized(),
                                     "BuildStrategy is finlaized.");
                      self.fuse_all_optimizer_ops_ = b;
                    })
Q
qingqing01 已提交
1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539
      .def_property(
          "sync_batch_norm",
          [](const BuildStrategy &self) { return self.sync_batch_norm_; },
          [](BuildStrategy &self, bool b) {
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
            self.sync_batch_norm_ = b;
          },
          R"DOC(The type is BOOL, sync_batch_norm indicates whether to use
                synchronous batch normalization which synchronizes the mean
                and variance through multi-devices in training phase.

                Current implementation doesn't support FP16 training and CPU.
                And only synchronous on one machine, not all machines.

F
flame 已提交
1540 1541 1542 1543 1544 1545 1546 1547 1548
                Default False

                Examples:
                    .. code-block:: python

                        import paddle.fluid as fluid
                        build_strategy = fluid.BuildStrategy()
                        build_strategy.sync_batch_norm = True
                )DOC")
D
dzhwinter 已提交
1549 1550
      .def_property(
          "memory_optimize",
1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569
          [](const BuildStrategy &self) -> py::object {
            if (self.memory_optimize_) {
              return py::cast(self.memory_optimize_.get());
            } else {
              return py::cast(nullptr);
            }
          },
          [](BuildStrategy &self, const py::handle &value) {
            auto *py_obj = value.ptr();
            if (py_obj == nullptr || py_obj == Py_None) {
              self.memory_optimize_ = boost::none;
            } else if (PyBool_Check(py_obj)) {
              self.memory_optimize_ = (py_obj == Py_True);
            } else {
              PADDLE_THROW(
                  "BuildStrategy.memory_optimize must be None, False or True");
            }
          },
          R"DOC(The type is BOOL or None, memory opitimize aims to save total memory
1570
                consumption, set to True to enable it.
1571

1572 1573 1574 1575
                Default None. None means framework would choose to use or not use 
                this strategy automatically. Currently, None means that it is 
                enabled when GC is disabled, and disabled when GC is enabled. 
                True means enabling and False means disabling. Default None.)DOC")
1576 1577 1578
      .def_property(
          "is_distribution",
          [](const BuildStrategy &self) { return self.is_distribution_; },
1579 1580 1581 1582 1583 1584 1585 1586 1587
          [](BuildStrategy &self, bool b) {
#ifdef WIN32
            if (b) {
              PADDLE_THROW("Windows has NO support to distribute mode.");
            }
#else
            self.is_distribution_ = b;
#endif
          })
Q
can run  
Qiao Longfei 已提交
1588 1589 1590
      .def_property("async_mode",
                    [](const BuildStrategy &self) { return self.async_mode_; },
                    [](BuildStrategy &self, bool b) { self.async_mode_ = b; })
D
dzhwinter 已提交
1591
      .def_property(
D
dzhwinter 已提交
1592 1593 1594
          "enable_inplace",
          [](const BuildStrategy &self) { return self.enable_inplace_; },
          [](BuildStrategy &self, bool b) { self.enable_inplace_ = b; })
C
chengduo 已提交
1595 1596 1597 1598
      .def_property(
          "fuse_all_reduce_ops",
          [](const BuildStrategy &self) { return self.fuse_all_reduce_ops_; },
          [](BuildStrategy &self, bool b) { self.fuse_all_reduce_ops_ = b; })
1599 1600 1601 1602 1603 1604 1605
      .def_property("enable_backward_optimizer_op_deps",
                    [](const BuildStrategy &self) {
                      return self.enable_backward_optimizer_op_deps_;
                    },
                    [](BuildStrategy &self, bool b) {
                      self.enable_backward_optimizer_op_deps_ = b;
                    })
1606 1607 1608 1609
      .def_property(
          "cache_runtime_context",
          [](const BuildStrategy &self) { return self.cache_runtime_context_; },
          [](BuildStrategy &self, bool b) { self.cache_runtime_context_ = b; })
1610 1611 1612 1613 1614 1615 1616 1617 1618
      .def_property(
          "mkldnn_enabled_op_types",
          [](const BuildStrategy &self) {
            return self.mkldnn_enabled_op_types_;
          },
          [](BuildStrategy &self,
             const std::unordered_set<std::string> &mkldnn_enabled_op_types) {
            self.mkldnn_enabled_op_types_ = mkldnn_enabled_op_types;
          })
1619
      .def("_finalize_strategy_and_create_passes",
X
fix  
Xin Pan 已提交
1620
           [](BuildStrategy &self) -> std::shared_ptr<ir::PassBuilder> {
1621 1622 1623 1624 1625
             return self.CreatePassesFromStrategy(true);
           },
           R"DOC(Allow user to customized passes. Normally model-specific
                optimization passes should be defined in this way. BuildStrategy
                cannot be updated after being finalized.)DOC");
Y
yuyang18 已提交
1626 1627

  pe.def(py::init<const std::vector<platform::Place> &,
Y
Yan Xu 已提交
1628
                  const std::vector<std::string> &, const std::string &,
X
Xin Pan 已提交
1629
                  Scope *, std::vector<Scope *> &, const ExecutionStrategy &,
X
Xin Pan 已提交
1630
                  const BuildStrategy &, ir::Graph *>())
Y
Yu Yang 已提交
1631 1632 1633 1634
      // NOTE: even we return a vec<Scope*>* to Python use reference policy.
      // We still cannot get local_scope from this vector, since the element
      // of vec<Scope*> will be freed by Python GC. We can only return Scope*
      // one by one and mark them as reference.
1635 1636 1637 1638 1639
      .def("local_scopes",
           [](ParallelExecutor &self) -> std::vector<Scope *> * {
             return &self.GetLocalScopes();
           },
           py::return_value_policy::reference)
1640 1641 1642
      .def("drop_local_exe_scopes", &ParallelExecutor::DropLocalExeScopes)
      .def("_need_create_local_exe_scopes",
           &ParallelExecutor::NeedCreateLocalExeScope)
Y
Yu Yang 已提交
1643 1644 1645 1646
      .def("feed_tensors_into_local_scopes",
           &ParallelExecutor::FeedTensorsIntoLocalScopes)
      .def("feed_and_split_tensor_into_local_scopes",
           &ParallelExecutor::FeedAndSplitTensorIntoLocalScopes)
S
sneaxiy 已提交
1647 1648 1649 1650 1651 1652
      .def("run", [](ParallelExecutor &self,
                     const std::vector<std::string> &fetch_tensors,
                     const std::string &fetched_var_name) {
        pybind11::gil_scoped_release release;
        self.Run(fetch_tensors, fetched_var_name);
      });
Y
Yu Yang 已提交
1653

1654
  BindRecordIOWriter(&m);
D
dongdaxiang 已提交
1655
  BindFleetWrapper(&m);
W
wopeizl 已提交
1656
#ifndef _WIN32
D
dongdaxiang 已提交
1657
  BindNCCLWrapper(&m);
W
wopeizl 已提交
1658
#endif
F
flame 已提交
1659 1660
  BindGraph(&m);
  BindNode(&m);
F
flame 已提交
1661
  BindInferenceApi(&m);
1662
  BindDataset(&m);
1663 1664 1665
#ifdef PADDLE_WITH_DISTRIBUTE
  BindCommunicator(&m);
#endif
L
Luo Tao 已提交
1666
}
1667
}  // namespace pybind
1668
}  // namespace paddle