pybind.cc 37.2 KB
Newer Older
1
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
2 3 4 5 6

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

7
http://www.apache.org/licenses/LICENSE-2.0
8 9 10 11 12 13

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
L
lgone2000 已提交
14
#include <Python.h>
C
chengduoZH 已提交
15 16
#include <algorithm>
#include <map>
S
sneaxiy 已提交
17
#include <memory>
C
chengduoZH 已提交
18 19 20 21 22
#include <mutex>  // NOLINT // for call_once
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>
23

Y
Yi Wang 已提交
24 25 26
#include "paddle/fluid/framework/executor.h"
#include "paddle/fluid/framework/feed_fetch_method.h"
#include "paddle/fluid/framework/framework.pb.h"
27
#include "paddle/fluid/framework/ir/pass_builder.h"
Y
Yi Wang 已提交
28 29 30
#include "paddle/fluid/framework/lod_rank_table.h"
#include "paddle/fluid/framework/lod_tensor.h"
#include "paddle/fluid/framework/lod_tensor_array.h"
31
#include "paddle/fluid/framework/op_registry.h"
Y
Yu Yang 已提交
32
#include "paddle/fluid/framework/parallel_executor.h"
Y
Yi Wang 已提交
33
#include "paddle/fluid/framework/prune.h"
Y
Refine  
Yu Yang 已提交
34
#include "paddle/fluid/framework/reader.h"
Y
Yi Wang 已提交
35
#include "paddle/fluid/framework/selected_rows.h"
X
Xin Pan 已提交
36
#include "paddle/fluid/framework/version.h"
D
dzhwinter 已提交
37
#include "paddle/fluid/operators/activation_op.h"
S
sneaxiy 已提交
38
#include "paddle/fluid/operators/reader/lod_tensor_blocking_queue.h"
Y
Yi Wang 已提交
39
#include "paddle/fluid/platform/enforce.h"
40
#include "paddle/fluid/platform/init.h"
Y
Yi Wang 已提交
41 42 43 44
#include "paddle/fluid/platform/place.h"
#include "paddle/fluid/platform/profiler.h"
#include "paddle/fluid/pybind/const_value.h"
#include "paddle/fluid/pybind/exception.h"
45 46
#include "paddle/fluid/pybind/protobuf.h"
#include "paddle/fluid/pybind/pybind.h"  // NOLINT
Y
Yu Yang 已提交
47
#include "paddle/fluid/pybind/recordio.h"
Y
Yi Wang 已提交
48
#include "paddle/fluid/pybind/tensor_py.h"
Y
Yu Yang 已提交
49

50
#include "paddle/fluid/string/to_string.h"
51

D
Dong Zhihong 已提交
52
#ifdef PADDLE_WITH_CUDA
Y
Yi Wang 已提交
53 54 55
#include "paddle/fluid/operators/nccl/nccl_gpu_common.h"
#include "paddle/fluid/platform/cuda_profiler.h"
#include "paddle/fluid/platform/gpu_info.h"
D
Dong Zhihong 已提交
56 57
#endif

M
minqiyang 已提交
58 59
#include "pybind11/stl.h"

60 61 62 63
DEFINE_bool(reader_queue_speed_test_mode, false,
            "If set true, the queue.pop will only get data from queue but not "
            "remove the data from queue for speed testing");

Q
Qiao Longfei 已提交
64 65 66
// disable auto conversion to list in Python
PYBIND11_MAKE_OPAQUE(paddle::framework::LoDTensorArray);

67
namespace paddle {
68
namespace pybind {
69
bool IsCompiledWithCUDA() {
70
#ifndef PADDLE_WITH_CUDA
Q
qijun 已提交
71 72 73 74 75 76
  return false;
#else
  return true;
#endif
}

Y
update  
Yancey1989 已提交
77
bool IsCompiledWithDIST() {
Y
Yancey1989 已提交
78
#ifdef PADDLE_WITH_DISTRIBUTE
Y
update  
Yancey1989 已提交
79 80 81 82 83 84
  return true;
#else
  return false;
#endif
}

85 86
PYBIND11_PLUGIN(core) {
  py::module m("core", "C++ core of PaddlePaddle");
87

88 89 90 91
  // using framework in this function. Since it is inside a function, it will
  // not cause namespace pollution.
  using namespace paddle::framework;  // NOLINT

92
  BindException(&m);
Y
Yu Yang 已提交
93

94 95 96
  py::class_<Tensor>(m, "Tensor", py::buffer_protocol())
      .def_buffer(
          [](Tensor &self) -> py::buffer_info { return CastToPyBuffer(self); })
Y
yuyang18 已提交
97
      .def("_get_dims",
98
           [](const Tensor &self) { return vectorize(self.dims()); })
Y
yuyang18 已提交
99
      .def("_set_dims",
Q
qijun 已提交
100
           [](Tensor &self, const std::vector<int64_t> &dim) {
Y
Yu Yang 已提交
101
             self.Resize(make_ddim(dim));
Y
Yu Yang 已提交
102
           })
Y
yuyang18 已提交
103
      .def("_set_layout",
D
dzhwinter 已提交
104 105 106
           [](Tensor &self, const std::string &layout) {
             self.set_layout(StringToDataLayout(layout));
           })
Y
yuyang18 已提交
107
      .def("_alloc_float",
D
dzhwinter 已提交
108
           [](Tensor &self, paddle::platform::CUDAPlace &place) {
Q
qijun 已提交
109
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
110
           })
Y
yuyang18 已提交
111
      .def("_alloc_float",
Y
Yu Yang 已提交
112
           [](Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
113
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
114
           })
Y
yuyang18 已提交
115
      .def("_alloc_int",
Y
Yu Yang 已提交
116
           [](Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
117
             self.mutable_data<int>(place);
Y
Yu Yang 已提交
118
           })
Y
yuyang18 已提交
119
      .def("_alloc_int",
D
dzhwinter 已提交
120
           [](Tensor &self, paddle::platform::CUDAPlace &place) {
Q
qijun 已提交
121
             self.mutable_data<int>(place);
Q
qijun 已提交
122
           })
Y
yuyang18 已提交
123
      .def("_alloc_int",
C
chengduoZH 已提交
124 125 126
           [](Tensor &self, paddle::platform::CUDAPinnedPlace &place) {
             self.mutable_data<int>(place);
           })
Y
yuyang18 已提交
127
      .def("_alloc_float",
C
chengduoZH 已提交
128 129 130
           [](Tensor &self, paddle::platform::CUDAPinnedPlace &place) {
             self.mutable_data<float>(place);
           })
Y
Yu Yang 已提交
131 132
      .def("set", PyCPUTensorSetFromArray<float>)
      .def("set", PyCPUTensorSetFromArray<int>)
133
      .def("set", PyCPUTensorSetFromArray<double>)
134
      .def("set", PyCPUTensorSetFromArray<int64_t>)
Y
Yu Yang 已提交
135
      .def("set", PyCPUTensorSetFromArray<bool>)
136
      .def("set", PyCPUTensorSetFromArray<uint16_t>)
F
fengjiayi 已提交
137
      .def("set", PyCPUTensorSetFromArray<uint8_t>)
Q
qingqing01 已提交
138
      .def("set", PyCPUTensorSetFromArray<int8_t>)
139
#ifdef PADDLE_WITH_CUDA
Y
Yu Yang 已提交
140 141
      .def("set", PyCUDATensorSetFromArray<float>)
      .def("set", PyCUDATensorSetFromArray<int>)
142
      .def("set", PyCUDATensorSetFromArray<double>)
143
      .def("set", PyCUDATensorSetFromArray<int64_t>)
Y
Yu Yang 已提交
144
      .def("set", PyCUDATensorSetFromArray<bool>)
145
      .def("set", PyCUDATensorSetFromArray<uint16_t>)
F
fengjiayi 已提交
146
      .def("set", PyCUDATensorSetFromArray<uint8_t>)
Q
qingqing01 已提交
147
      .def("set", PyCUDATensorSetFromArray<int8_t>)
C
chengduoZH 已提交
148 149 150 151 152 153
      .def("set", PyCUDAPinnedTensorSetFromArray<float>)
      .def("set", PyCUDAPinnedTensorSetFromArray<int>)
      .def("set", PyCUDAPinnedTensorSetFromArray<double>)
      .def("set", PyCUDAPinnedTensorSetFromArray<int64_t>)
      .def("set", PyCUDAPinnedTensorSetFromArray<bool>)
      .def("set", PyCUDAPinnedTensorSetFromArray<uint16_t>)
F
fengjiayi 已提交
154
      .def("set", PyCUDAPinnedTensorSetFromArray<uint8_t>)
Q
qingqing01 已提交
155
      .def("set", PyCUDAPinnedTensorSetFromArray<int8_t>)
Q
qijun 已提交
156
#endif
157
      .def("shape", [](Tensor &self) { return vectorize(self.dims()); })
Y
yuyang18 已提交
158 159 160 161 162
      .def("_set_float_element", TensorSetElement<float>)
      .def("_get_float_element", TensorGetElement<float>)
      .def("_set_double_element", TensorSetElement<double>)
      .def("_get_double_element", TensorGetElement<double>)
      .def("_dtype", [](Tensor &self) { return ToDataType(self.type()); });
Y
Yu Yang 已提交
163

X
Xin Pan 已提交
164 165 166 167 168 169 170 171 172 173 174 175 176
  py::class_<LoDTensor, Tensor>(m, "LoDTensor", R"DOC(
    LoDTensor is a Tensor with optional LoD information.

    np.array(lod_tensor) can convert LoDTensor to numpy array.
    lod_tensor.lod() can retrieve the LoD information.

    LoD is short for Level of Details and is usually used for varied sequence
    length. You can skip the following comment if you don't need optional LoD.

  For example:
     A LoDTensor X can look like the example below. It contains 2 sequences.
     The first has length 2 and the second has length 3, as described by x.lod.

X
fix doc  
Xin Pan 已提交
177
     The first tensor dimension 5=2+3 is calculated from LoD if it's available.
X
Xin Pan 已提交
178
     It means the total number of sequence element. In X, each element has 2
X
fix doc  
Xin Pan 已提交
179
     columns, hence [5, 2].
X
Xin Pan 已提交
180 181 182

      x.lod  = [[2, 3]]
      x.data = [[1, 2], [3, 4],
X
fix doc  
Xin Pan 已提交
183 184
                [5, 6], [7, 8], [9, 10]]
      x.shape = [5, 2]
X
Xin Pan 已提交
185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207

      LoD can have multiple levels (for example, a paragraph can have multiple
      sentences and a sentence can have multiple words). In the following
      LodTensor Y, the lod_level is 2. It means there are 2 sequence, the
      first sequence length is 2 (has 2 sub-sequences), the second one's
      length is 1. The first sequence's 2 sub-sequences have length 2 and 2,
      respectively. And the second sequence's 1 sub-sequence has length 3.

      y.lod = [[2 1], [2 2 3]]
      y.shape = [2+2+3, ...]

  Note:
      In above description, LoD is length-based. In Paddle internal
      implementation, lod is offset-based. Hence, internally,
      y.lod is represented as [[0, 2, 3], [0, 2, 4, 7]] (length-based
      equivlent would be [[2-0, 3-2], [2-0, 4-2, 7-4]]).

      Sometimes LoD is called recursive_sequence_length to be more
      self-explanatory. In this case, it must be length-based. Due to history
      reasons. when LoD is called lod in public API, it might be offset-based.
      Users should be careful about it.

        )DOC")
208 209
      .def_buffer(
          [](Tensor &self) -> py::buffer_info { return CastToPyBuffer(self); })
210 211 212 213 214 215 216 217 218 219 220 221 222 223
      .def("__init__",
           [](LoDTensor &instance, const std::vector<std::vector<size_t>>
                                       &recursive_sequence_lengths) {
             LoD new_lod;
             new_lod.reserve(recursive_sequence_lengths.size());
             std::copy(recursive_sequence_lengths.begin(),
                       recursive_sequence_lengths.end(),
                       std::back_inserter(new_lod));
             LoD new_offset_lod = ConvertToOffsetBasedLoD(new_lod);
             PADDLE_ENFORCE(
                 CheckLoD(new_offset_lod, -1),
                 "the provided recursive_sequence_lengths info is invalid");
             new (&instance) LoDTensor(new_offset_lod);
           })
Y
Yu Yang 已提交
224
      .def("__init__", [](LoDTensor &instance) { new (&instance) LoDTensor(); })
G
gongweibao 已提交
225 226 227 228 229
      // We implement offset based LOD in C++ while we use length based with
      // Python API. So we changed set_lod to set_recursive_sequence_lengths to
      // avoid misuse.
      // The discussion is here:
      // https://github.com/PaddlePaddle/Paddle/issues/10855
D
dangqingqing 已提交
230
      .def("set_lod",
231
           [](LoDTensor &self, const std::vector<std::vector<size_t>> &lod) {
232
             // the input lod is offset-based level-of-detail info
Y
Yu Yang 已提交
233
             LoD new_lod;
234 235
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
236 237
             PADDLE_ENFORCE(CheckLoD(new_lod, vectorize(self.dims()).front()),
                            "the provided lod info is invalid");
238
             self.set_lod(new_lod);
D
dangqingqing 已提交
239
           })
240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264
      .def("set_recursive_sequence_lengths",
           [](LoDTensor &self, const std::vector<std::vector<size_t>>
                                   &recursive_sequence_lengths) {
             // the input recursive_sequence_lengths is length-based
             // level-of-detail info
             LoD new_lod;
             new_lod.reserve(recursive_sequence_lengths.size());
             std::copy(recursive_sequence_lengths.begin(),
                       recursive_sequence_lengths.end(),
                       std::back_inserter(new_lod));
             LoD new_offset_lod = ConvertToOffsetBasedLoD(new_lod);
             PADDLE_ENFORCE(
                 CheckLoD(new_offset_lod, vectorize(self.dims()).front()),
                 "the provided recursive_sequence_lengths info is invalid");
             self.set_lod(new_offset_lod);
           })
      .def("lod",
           [](LoDTensor &self) -> std::vector<std::vector<size_t>> {
             // output the offset-based lod info
             LoD lod = self.lod();
             std::vector<std::vector<size_t>> new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             return new_lod;
           })
G
gongweibao 已提交
265
      // Set above comments of set_lod.
266 267 268 269 270 271 272 273 274 275 276 277 278
      .def("recursive_sequence_lengths",
           [](LoDTensor &self) -> std::vector<std::vector<size_t>> {
             // output the length-based lod info
             LoD lod = ConvertToLengthBasedLoD(self.lod());
             std::vector<std::vector<size_t>> new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             return new_lod;
           })
      .def("has_valid_recursive_sequence_lengths", [](LoDTensor &self) -> bool {
        // Check that the lod info is valid and match the outermost
        // dimension of the LoDTensor data
        return CheckLoD(self.lod(), vectorize(self.dims()).front());
D
dangqingqing 已提交
279 280
      });

Q
qijun 已提交
281 282 283 284 285 286 287 288 289 290 291 292 293
  py::class_<SelectedRows>(m, "SelectedRows")
      .def("__init__",
           [](SelectedRows &instance) { new (&instance) SelectedRows(); })
      .def("__init__",
           [](SelectedRows &instance, const std::vector<int64_t> rows,
              const int64_t &height) {
             new (&instance) SelectedRows(rows, height);
           })
      .def("get_tensor",
           [](SelectedRows &self) { return self.mutable_value(); },
           py::return_value_policy::reference)
      .def("set_height", &SelectedRows::set_height)
      .def("height", &SelectedRows::height)
Q
qijun 已提交
294 295 296 297 298 299 300 301 302
      .def("set_rows",
           [](SelectedRows &self, std::vector<int64_t> rows) {
#ifndef PADDLE_WITH_CUDA
             self.set_rows(rows);
#else
        Vector<int64_t> new_rows(rows);
        self.set_rows(new_rows);
#endif
           })
303
      .def("sync_index", [](SelectedRows &instance) { instance.SyncIndex(); })
304
      .def("rows", [](SelectedRows &self) {
305 306 307 308 309
        auto rows = self.rows();
        std::vector<int64_t> new_rows;
        new_rows.reserve(rows.size());
        std::copy(rows.begin(), rows.end(), std::back_inserter(new_rows));
        return new_rows;
310
      });
Q
qijun 已提交
311

312
  py::class_<Variable>(m, "Variable", R"DOC(Variable Class.
313 314 315

All parameter, weight, gradient are variables in Paddle.
)DOC")
316
      .def("is_int", [](const Variable &var) { return var.IsType<int>(); })
317
      .def("set_int",
318 319
           [](Variable &var, int val) -> void { *var.GetMutable<int>() = val; })
      .def("get_int", [](const Variable &var) -> int { return var.Get<int>(); })
320 321 322 323 324 325 326
      .def("is_float", [](const Variable &var) { return var.IsType<float>(); })
      .def("set_float",
           [](Variable &var, float val) -> void {
             *var.GetMutable<float>() = val;
           })
      .def("get_float",
           [](const Variable &var) -> float { return var.Get<float>(); })
Y
Yu Yang 已提交
327
      .def("get_tensor",
328 329
           [](Variable &self) -> LoDTensor * {
             return self.GetMutable<LoDTensor>();
D
dangqingqing 已提交
330 331
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
332 333 334
      .def("get_lod_rank_table",
           [](Variable &self) { return self.GetMutable<LoDRankTable>(); },
           py::return_value_policy::reference)
Q
qijun 已提交
335 336 337 338 339
      .def("get_selected_rows",
           [](Variable &self) -> SelectedRows * {
             return self.GetMutable<SelectedRows>();
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
340 341 342
      .def("get_lod_tensor_array",
           [](Variable &self) { return self.GetMutable<LoDTensorArray>(); },
           py::return_value_policy::reference)
D
Dong Zhihong 已提交
343 344 345 346 347 348 349
#ifdef PADDLE_WITH_CUDA
      .def("get_communicator",
           [](Variable &self) -> platform::Communicator * {
             return self.GetMutable<platform::Communicator>();
           },
           py::return_value_policy::reference)
#endif
Y
Refine  
Yu Yang 已提交
350 351 352 353 354
      .def("get_reader",
           [](Variable &self) -> framework::ReaderHolder * {
             PADDLE_ENFORCE(self.IsType<framework::ReaderHolder>());
             return self.GetMutable<framework::ReaderHolder>();
           },
Y
Yu Yang 已提交
355
           py::return_value_policy::reference);
356

Y
Refine  
Yu Yang 已提交
357
  py::class_<framework::ReaderHolder>(m, "Reader", "")
358
      .def("reset", &framework::ReaderHolder::ResetAll);
Y
Refine  
Yu Yang 已提交
359

S
sneaxiy 已提交
360 361 362 363
  using LoDTensorBlockingQueue =
      ::paddle::operators::reader::LoDTensorBlockingQueue;
  using LoDTensorBlockingQueueHolder =
      ::paddle::operators::reader::LoDTensorBlockingQueueHolder;
S
sneaxiy 已提交
364 365
  py::class_<LoDTensorBlockingQueue, std::shared_ptr<LoDTensorBlockingQueue>>(
      m, "LoDTensorBlockingQueue", "")
S
sneaxiy 已提交
366
      .def("push",
S
sneaxiy 已提交
367
           [](LoDTensorBlockingQueue &self,
S
sneaxiy 已提交
368
              const std::vector<framework::LoDTensor> &lod_tensor_vec) {
S
sneaxiy 已提交
369
             pybind11::gil_scoped_release release;
S
sneaxiy 已提交
370
             return self.Push(lod_tensor_vec);
S
sneaxiy 已提交
371
           })
S
sneaxiy 已提交
372 373 374 375
      .def("size", &LoDTensorBlockingQueue::Size)
      .def("capacity", &LoDTensorBlockingQueue::Cap)
      .def("close", &LoDTensorBlockingQueue::Close)
      .def("is_closed", &LoDTensorBlockingQueue::IsClosed);
S
sneaxiy 已提交
376

S
sneaxiy 已提交
377
  m.def("init_lod_tensor_blocking_queue",
S
sneaxiy 已提交
378
        [](Variable &var, size_t capacity,
S
sneaxiy 已提交
379
           const std::vector<std::vector<int64_t>> &shapes)
S
sneaxiy 已提交
380
            -> std::shared_ptr<LoDTensorBlockingQueue> {
S
sneaxiy 已提交
381 382 383 384 385 386
              std::vector<DDim> dims(shapes.size());
              std::transform(shapes.begin(), shapes.end(), dims.begin(),
                             [](const std::vector<int64_t> &shape) {
                               return make_ddim(shape);
                             });
              auto *holder = var.GetMutable<LoDTensorBlockingQueueHolder>();
387 388
              holder->InitOnce(capacity, dims,
                               FLAGS_reader_queue_speed_test_mode);
S
sneaxiy 已提交
389
              return holder->GetQueue();
S
sneaxiy 已提交
390
            },
S
sneaxiy 已提交
391
        py::return_value_policy::copy);
S
sneaxiy 已提交
392

393
  py::class_<Scope>(m, "Scope", "")
D
dongzhihong 已提交
394
      .def("var",
395
           [](Scope &self, const std::string &name) -> Variable * {
D
dongzhihong 已提交
396
             return self.Var(name);
Y
Yu Yang 已提交
397
           },
398
           py::return_value_policy::reference)
399
      .def("find_var", &Scope::FindVar, py::return_value_policy::reference)
Y
Yu Yang 已提交
400
      .def(py::init<>())
401
      .def("new_scope", [](Scope &self) -> Scope * { return &self.NewScope(); },
402
           py::return_value_policy::reference)
Y
Yu Yang 已提交
403
      .def("drop_kids", &Scope::DropKids);
404

Y
Yu Yang 已提交
405 406
  //! @note: Be careful! PyBind will return std::string as an unicode, not
  //! Python str. If you want a str object, you should cast them in Python.
Y
Yu Yang 已提交
407 408
  m.def("get_all_op_protos", []() -> std::vector<py::bytes> {
    std::vector<py::bytes> ret_values;
409 410 411 412 413 414 415 416 417 418
    for (auto &iter : OpInfoMap::Instance().map()) {
      auto &info = iter.second;
      if (info.HasOpProtoAndChecker()) {
        std::string str;
        PADDLE_ENFORCE(
            info.Proto().SerializeToString(&str),
            "Serialize OpProto Error. This could be a bug of Paddle.");
        ret_values.emplace_back(str);
      }
    }
Y
Yu Yang 已提交
419 420
    return ret_values;
  });
421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436
  m.def(
      "get_grad_op_desc", [](const OpDesc &op_desc,
                             const std::unordered_set<std::string> &no_grad_set,
                             const std::vector<BlockDesc *> &grad_sub_block) {
        std::unordered_map<std::string, std::string> grad_to_var;
        std::vector<std::unique_ptr<OpDesc>> grad_op_descs =
            framework::OpInfoMap::Instance()
                .Get(op_desc.Type())
                .GradOpMaker()(op_desc, no_grad_set, &grad_to_var,
                               grad_sub_block);
        std::vector<OpDesc *> grad_op_desc_ptrs(grad_op_descs.size());
        std::transform(grad_op_descs.begin(), grad_op_descs.end(),
                       grad_op_desc_ptrs.begin(),
                       [](std::unique_ptr<OpDesc> &p) { return p.release(); });
        return std::make_pair(grad_op_desc_ptrs, grad_to_var);
      });
Y
Yu Yang 已提交
437
  m.def("prune", [](const ProgramDesc &origin,
438
                    const std::vector<std::array<size_t, 2>> &targets) {
Y
Yu Yang 已提交
439
    ProgramDesc prog_with_targets(origin);
440
    for (const auto &t : targets) {
441
      prog_with_targets.MutableBlock(t[0])->Op(t[1])->SetIsTarget(true);
442
    }
443
    proto::ProgramDesc pruned_desc;
444
    Prune(*prog_with_targets.Proto(), &pruned_desc);
Y
Yu Yang 已提交
445
    return new ProgramDesc(pruned_desc);
446
  });
447 448 449 450
  m.def("empty_var_name",
        []() { return std::string(framework::kEmptyVarName); });
  m.def("grad_var_suffix",
        []() { return std::string(framework::kGradVarSuffix); });
451 452 453
  m.def_submodule(
       "var_names",
       "The module will return special predefined variable name in Paddle")
Y
Yi Wang 已提交
454 455
      .def("empty", []() { return kEmptyVarName; })
      .def("temp", []() { return kTempVarName; });
Q
qijun 已提交
456
  // clang-format off
Y
Yu Yang 已提交
457
  py::class_<paddle::platform::DeviceContext>(m, "DeviceContext")
Q
qijun 已提交
458 459
      .def_static("create",
                  [](paddle::platform::CPUPlace& place)
Q
qijun 已提交
460
                      -> paddle::platform::DeviceContext* {
Q
qijun 已提交
461 462 463
                    return new paddle::platform::CPUDeviceContext();
                  })
      .def_static("create",
D
dzhwinter 已提交
464
                  [](paddle::platform::CUDAPlace& place)
Q
qijun 已提交
465
                      -> paddle::platform::DeviceContext* {
466
#ifndef PADDLE_WITH_CUDA
D
dzhwinter 已提交
467
                    PADDLE_THROW("CUDAPlace is not supported in CPU device.");
Q
qijun 已提交
468
#else
Q
qijun 已提交
469
                    return new paddle::platform::CUDADeviceContext(place);
Q
qijun 已提交
470
#endif
C
chengduoZH 已提交
471 472 473 474 475 476 477 478 479 480 481
                  })
          .def_static("create",
                [](paddle::platform::CUDAPinnedPlace& place)
                        -> paddle::platform::DeviceContext* {
#ifndef PADDLE_WITH_CUDA
                  PADDLE_THROW(
                        "CUDAPinnedPlace is not supported in CPU device.");
#else
                  return new paddle::platform::CUDAPinnedDeviceContext(place);
#endif
                });;
D
Dong Zhihong 已提交
482 483 484 485
// clang-format on
#ifdef PADDLE_WITH_CUDA
  py::class_<platform::Communicator>(m, "Communicator").def(py::init<>());
#endif
D
dzhwinter 已提交
486
  py::class_<platform::CUDAPlace>(m, "CUDAPlace")
487
      .def(py::init<int>())
D
dzhwinter 已提交
488
      .def("__str__", string::to_string<const platform::CUDAPlace &>);
Q
qijun 已提交
489

490 491 492
  py::class_<paddle::platform::CPUPlace>(m, "CPUPlace")
      .def(py::init<>())
      .def("__str__", string::to_string<const platform::CPUPlace &>);
Y
Yu Yang 已提交
493

C
chengduoZH 已提交
494 495 496 497
  py::class_<paddle::platform::CUDAPinnedPlace>(m, "CUDAPinnedPlace")
      .def(py::init<>())
      .def("__str__", string::to_string<const platform::CUDAPinnedPlace &>);

Y
Yu Yang 已提交
498 499 500 501 502 503 504
  py::class_<platform::Place>(m, "Place")
      .def(py::init<>())
      .def("set_place",
           [](platform::Place &self, const platform::CPUPlace &cpu_place) {
             self = cpu_place;
           })
      .def("set_place",
D
dzhwinter 已提交
505
           [](platform::Place &self, const platform::CUDAPlace &gpu_place) {
Y
Yu Yang 已提交
506
             self = gpu_place;
C
chengduoZH 已提交
507 508
           })
      .def("set_place", [](platform::Place &self,
C
chengduoZH 已提交
509 510
                           const platform::CUDAPinnedPlace &cuda_pinned_place) {
        self = cuda_pinned_place;
C
chengduoZH 已提交
511
      });
Y
Yu Yang 已提交
512

Y
Yu Yang 已提交
513 514 515
  py::class_<OperatorBase>(m, "Operator")
      .def_static("create",
                  [](py::bytes protobin) {
516
                    proto::OpDesc desc;
Y
Yu Yang 已提交
517 518 519 520 521
                    PADDLE_ENFORCE(desc.ParsePartialFromString(protobin),
                                   "Cannot parse user input to OpDesc");
                    PADDLE_ENFORCE(desc.IsInitialized(),
                                   "User OpDesc is not initialized, reason %s",
                                   desc.InitializationErrorString());
522
                    return OpRegistry::CreateOp(desc);
Y
Yu Yang 已提交
523
                  })
524
      .def("run",
525
           [](OperatorBase &self, const Scope &scope,
D
dzhwinter 已提交
526 527 528
              const platform::CPUPlace &place) { self.Run(scope, place); })
      .def("run",
           [](OperatorBase &self, const Scope &scope,
D
dzhwinter 已提交
529
              const platform::CUDAPlace &place) { self.Run(scope, place); })
C
chengduoZH 已提交
530 531 532 533 534
      .def("run",
           [](OperatorBase &self, const Scope &scope,
              const platform::CUDAPinnedPlace &place) {
             self.Run(scope, place);
           })
Y
Yu Yang 已提交
535 536 537 538 539 540 541
      .def("type",
           [](const OperatorBase &op) -> std::string { return op.Type(); })
      .def("outputs",
           [](const OperatorBase &op)
               -> std::map<std::string, std::vector<std::string>> {
                 return op.Outputs();
               })
Q
qijun 已提交
542 543
      .def("output_vars",
           [](const OperatorBase &op) { return op.OutputVars(true); })
Y
Yu Yang 已提交
544
      .def("inputs", [](const OperatorBase &op) { return op.Inputs(); })
Q
qijun 已提交
545
      .def("input_vars", [](const OperatorBase &op) { return op.InputVars(); })
Y
Yu Yang 已提交
546 547 548 549
      .def("__str__", &OperatorBase::DebugString)
      .def("no_intermediate_outputs",
           [](const OperatorBase &op) { return op.OutputVars(false); })
      .def("support_gpu", &OperatorBase::SupportGPU);
Y
Yu Yang 已提交
550

F
fengjiayi 已提交
551
  py::class_<framework::Executor>(m, "Executor")
D
dzhwinter 已提交
552
      .def(py::init<const platform::Place &>())
Y
Yancey1989 已提交
553
      .def("close", &Executor::Close)
S
sneaxiy 已提交
554 555 556 557 558
      .def("run", [](Executor &self, const ProgramDesc &prog, Scope *scope,
                     int block_id, bool create_local_scope, bool create_vars) {
        pybind11::gil_scoped_release release;
        self.Run(prog, scope, block_id, create_local_scope, create_vars);
      });
S
sneaxiy 已提交
559

D
dzhwinter 已提交
560
  m.def("init_gflags", framework::InitGflags);
Y
Yang Yu 已提交
561
  m.def("init_glog", framework::InitGLOG);
X
Xin Pan 已提交
562 563
  m.def("init_devices",
        [](bool init_p2p) { framework::InitDevices(init_p2p); });
564

565
  m.def("is_compiled_with_cuda", IsCompiledWithCUDA);
Y
update  
Yancey1989 已提交
566
  m.def("is_compiled_with_dist", IsCompiledWithDIST);
567 568 569 570 571 572
#ifdef PADDLE_WITH_CUDA
  m.def("is_float16_supported", [](const platform::CUDAPlace &place) -> bool {
    // Only GPUs with Compute Capability >= 53 support float16
    return platform::GetCUDAComputeCapability(place.device) >= 53;
  });
#endif
573

574
  m.def("set_feed_variable", framework::SetFeedVariable);
Q
qijun 已提交
575
  m.def("get_fetch_variable", framework::GetFetchVariable);
Q
qijun 已提交
576

X
Xin Pan 已提交
577 578
  m.def("_is_program_version_supported", IsProgramVersionSupported);

579 580 581 582 583
  BindProgramDesc(&m);
  BindBlockDesc(&m);
  BindVarDsec(&m);
  BindOpDesc(&m);
  BindConstValue(&m);
Y
Yu Yang 已提交
584

Y
Yu Yang 已提交
585 586 587 588 589 590 591 592 593
  py::class_<framework::LoDRankTable>(m, "LodRankTable")
      .def("items", [](framework::LoDRankTable &table) {
        std::vector<std::pair<size_t, size_t>> res;
        for (auto &item : table.items()) {
          res.push_back({item.index, item.length});
        }
        return res;
      });

Y
Yu Yang 已提交
594
  py::class_<LoDTensorArray>(m, "LoDTensorArray")
S
sneaxiy 已提交
595 596
      .def("__init__",
           [](LoDTensorArray &instance) { new (&instance) LoDTensorArray(); })
Y
Yu Yang 已提交
597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612
      .def("__getitem__",
           [](LoDTensorArray &self, size_t i) { return &self.at(i); },
           py::return_value_policy::reference)
      .def("__len__", [](LoDTensorArray &self) { return self.size(); })
      .def("__setitem__",
           [](LoDTensorArray &self, size_t i, const LoDTensor &t) {
             PADDLE_ENFORCE_LT(i, self.size());
             self[i].ShareDataWith(t);
             self[i].set_lod(t.lod());
           })
      .def("append", [](LoDTensorArray &self, const LoDTensor &t) {
        self.emplace_back();
        self.back().ShareDataWith(t);
        self.back().set_lod(t.lod());
      });

D
dzhwinter 已提交
613 614 615
  m.def("IsInplace",
        [](std::string op) -> bool { return operators::IsInplace(op); });

Y
Yu Yang 已提交
616
  m.def("op_support_gpu", OpSupportGPU);
D
Dong Zhihong 已提交
617
#ifdef PADDLE_WITH_CUDA
D
Dong Zhihong 已提交
618
  m.def("get_cuda_device_count", platform::GetCUDADeviceCount);
D
dangqingqing 已提交
619 620 621 622

  m.def("nvprof_init", platform::CudaProfilerInit);
  m.def("nvprof_start", platform::CudaProfilerStart);
  m.def("nvprof_stop", platform::CudaProfilerStop);
D
Dong Zhihong 已提交
623
#endif
Y
Yu Yang 已提交
624

625 626 627 628
  py::enum_<platform::ProfilerState>(m, "ProfilerState", py::arithmetic())
      .value("kDisabled", platform::ProfilerState::kDisabled)
      .value("kCPU", platform::ProfilerState::kCPU)
      .value("kCUDA", platform::ProfilerState::kCUDA)
629
      .value("kAll", platform::ProfilerState::kAll)
630 631 632 633 634 635 636 637 638 639 640 641 642
      .export_values();

  py::enum_<platform::EventSortingKey>(m, "EventSortingKey", py::arithmetic())
      .value("kDefault", platform::EventSortingKey::kDefault)
      .value("kCalls", platform::EventSortingKey::kCalls)
      .value("kTotal", platform::EventSortingKey::kTotal)
      .value("kMin", platform::EventSortingKey::kMin)
      .value("kMax", platform::EventSortingKey::kMax)
      .value("kAve", platform::EventSortingKey::kAve)
      .export_values();

  m.def("enable_profiler", platform::EnableProfiler);
  m.def("disable_profiler", platform::DisableProfiler);
X
Xin Pan 已提交
643
  m.def("is_profiler_enabled", platform::IsProfileEnabled);
644
  m.def("reset_profiler", platform::ResetProfiler);
Y
Yu Yang 已提交
645

646 647
  py::class_<ir::Pass, std::shared_ptr<ir::Pass>> pass(m, "Pass");
  pass.def(py::init())
648 649 650 651 652 653 654
      .def(
          "set_str",
          [](ir::Pass &self, const std::string &name, const std::string &attr) {
            self.Set<std::string>(name, new std::string(attr));
          })
      .def("set_int", [](ir::Pass &self, const std::string &name, int val) {
        self.Set<const int>(name, new int(val));
655 656
      });

X
fix  
Xin Pan 已提交
657 658
  py::class_<ir::PassBuilder, std::shared_ptr<ir::PassBuilder>> pb(
      m, "PassBuilder");
659 660 661 662 663 664 665 666 667 668 669 670 671 672
  pb.def(py::init())
      .def("append_pass",
           [](ir::PassBuilder &self,
              const std::string &pass_type) -> std::shared_ptr<ir::Pass> {
             return self.AppendPass(pass_type);
           })
      .def("all_passes", [](ir::PassBuilder &self) { return self.AllPasses(); })
      .def("insert_pass",
           [](ir::PassBuilder &self, size_t idx, const std::string &pass_type) {
             return self.InsertPass(idx, pass_type);
           })
      .def("remove_pass",
           [](ir::PassBuilder &self, size_t idx) { self.RemovePass(idx); });

Y
yuyang18 已提交
673
  // -- python binds for parallel executor.
Y
yuyang18 已提交
674
  py::class_<ParallelExecutor> pe(m, "ParallelExecutor");
C
chengduo 已提交
675 676 677 678
  py::class_<ExecutionStrategy> exec_strategy(pe, "ExecutionStrategy", R"DOC(
    ExecutionStrategy allows the user to more preciously control how to run
    the program in ParallelExecutor by setting the property.

C
chengduo 已提交
679 680 681 682 683 684 685 686 687 688 689
    Examples:
        .. code-block:: python

          exec_strategy = fluid.ExecutionStrategy()
          exec_strategy.num_threads = 4

          train_exe = fluid.ParallelExecutor(use_cuda=True,
                                             loss_name=loss.name,
                                             exec_strategy=exec_strategy)

          train_loss, = train_exe.run([loss.name], feed=feed_dict)
C
chengduo 已提交
690 691 692

        )DOC");

Y
yuyang18 已提交
693
  exec_strategy.def(py::init())
Y
yuyang18 已提交
694 695 696 697 698
      .def_property(
          "num_threads",
          [](const ExecutionStrategy &self) { return self.num_threads_; },
          [](ExecutionStrategy &self, size_t num_threads) {
            self.num_threads_ = num_threads;
C
chengduo 已提交
699 700 701 702 703 704 705 706 707 708
          },
          R"DOC(The type is INT, num_threads represents the size of thread pool that
            used to run the operators of the current program in ParallelExecutor.
            If :math:`num\_threads=1`, all the operators will execute one by one,
            but the order maybe difference between iterations.
            If it is not set, it will be set in ParallelExecutor according to the
            device type and device count, for GPU, :math:`num\_threads=device\_count*4`, for CPU,
            :math:`num\_threads=CPU\_NUM*4`, the explanation of:math:`CPU\_NUM` is in ParallelExecutor.
            if it is not set, ParallelExecutor will get the cpu count by calling
            `multiprocessing.cpu_count()`. Default 0.)DOC")
Y
yuyang18 已提交
709
      .def_property(
710 711 712 713
          "use_cuda",
          [](const ExecutionStrategy &self) { return self.use_cuda_; },
          [](ExecutionStrategy &self, bool use_cuda) {
            self.use_cuda_ = use_cuda;
C
chengduo 已提交
714 715 716 717
          })  // FIXME(chengduo): Doesn't add doc for 'use_cuda', use_cuda may
      // make user confuse, because ParallelExecutor has a parameter named
      // 'use_cuda' too, in current implementation, ParallelExecutor's
      // 'use_cuda' will rewrite ExecutionStrategy's 'use_cuda'.
Y
yuyang18 已提交
718 719 720 721 722
      .def_property(
          "allow_op_delay",
          [](const ExecutionStrategy &self) { return self.allow_op_delay_; },
          [](ExecutionStrategy &self, bool allow_op_delay) {
            self.allow_op_delay_ = allow_op_delay;
C
chengduo 已提交
723 724 725 726
          },
          R"DOC(The type is BOOL, allow_op_delay represents whether to delay the
                communication operators to run, it may make the execution faster.
                Note that in some models, allow_op_delay may cause program hang. Default False.)DOC")
Y
yuyang18 已提交
727 728 729 730 731 732 733
      .def_property(
          "num_iteration_per_drop_scope",
          [](const ExecutionStrategy &self) {
            return self.num_iteration_per_drop_scope_;
          },
          [](ExecutionStrategy &self, size_t num_iteration_per_drop_scope) {
            self.num_iteration_per_drop_scope_ = num_iteration_per_drop_scope;
C
chengduo 已提交
734 735 736 737 738 739 740 741 742 743 744 745 746
          },
          R"DOC(The type is INT, num_iteration_per_drop_scope indicates how
                many iterations to clean up the temp variables which
                is generated during execution. It may make the execution faster,
                because the temp variable's shape maybe the same between two iterations. Default 100.

                NOTES:
                    1. If you fetch data when calling the 'run', the ParallelExecutor
                       will clean up the temp variables at the end of the current iteration.
                    2. In some NLP model, it may cause the GPU memory is insufficient,
                       in this case, you should reduce `num_iteration_per_drop_scope`.
              )DOC");

Y
yuyang18 已提交
747
  exec_strategy.def_property(
Y
yuyang18 已提交
748 749 750 751 752 753 754
      "use_experimental_executor",
      [](const ExecutionStrategy &self) {
        return self.type_ == ExecutionStrategy::kExperimental;
      },
      [](ExecutionStrategy &self, bool experimental) {
        self.type_ = experimental ? ExecutionStrategy::kExperimental
                                  : ExecutionStrategy::kDefault;
Y
yuyang18 已提交
755 756
      });

C
chengduo 已提交
757 758 759 760
  py::class_<BuildStrategy> build_strategy(pe, "BuildStrategy", R"DOC(
    BuildStrategy allows the user to more preciously control how to
    build the SSA Graph in ParallelExecutor by setting the property.

C
chengduo 已提交
761 762 763 764 765 766 767 768 769 770 771
    Examples:
        .. code-block:: python

          build_strategy = fluid.BuildStrategy()
          build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce

          train_exe = fluid.ParallelExecutor(use_cuda=True,
                                             loss_name=loss.name,
                                             build_strategy=build_strategy)

          train_loss, = train_exe.run([loss.name], feed=feed_dict)
C
chengduo 已提交
772
)DOC");
Y
yuyang18 已提交
773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789

  py::enum_<BuildStrategy::ReduceStrategy>(build_strategy, "ReduceStrategy")
      .value("Reduce", BuildStrategy::ReduceStrategy::kReduce)
      .value("AllReduce", BuildStrategy::ReduceStrategy::kAllReduce);
  py::enum_<BuildStrategy::GradientScaleStrategy>(build_strategy,
                                                  "GradientScaleStrategy")
      .value("CoeffNumDevice",
             BuildStrategy::GradientScaleStrategy::kCoeffNumDevice)
      .value("One", BuildStrategy::GradientScaleStrategy::kOne)
      .value("Customized", BuildStrategy::GradientScaleStrategy::kCustomized);

  build_strategy.def(py::init())
      .def_property(
          "reduce_strategy",
          [](const BuildStrategy &self) { return self.reduce_; },
          [](BuildStrategy &self, BuildStrategy::ReduceStrategy strategy) {
            self.reduce_ = strategy;
C
chengduo 已提交
790 791 792 793 794 795 796
          },
          R"DOC(The type is STR, there are two reduce strategies in ParallelExecutor,
                  'AllReduce' and 'Reduce'. If you want that all the parameters'
                  optimization are done on all devices independently, you should choose 'AllReduce';
                  if you choose 'Reduce', all the parameters' optimization will be evenly distributed
                  to different devices, and then broadcast the optimized parameter to other devices.
                  In some models, `Reduce` is faster. Default 'AllReduce'. )DOC")
Y
yuyang18 已提交
797 798 799 800 801 802
      .def_property(
          "gradient_scale_strategy",
          [](const BuildStrategy &self) { return self.gradient_scale_; },
          [](BuildStrategy &self,
             BuildStrategy::GradientScaleStrategy strategy) {
            self.gradient_scale_ = strategy;
C
chengduo 已提交
803 804 805 806 807 808
          },
          R"DOC(The type is STR, there are three ways of defining :math:`loss@grad` in
                   ParallelExecutor, 'CoeffNumDevice', 'One' and 'Customized'. By default,
                   ParallelExecutor sets the :math:`loss@grad` according to the number of devices.
                   If you want to customize :math:`loss@grad`, you can choose 'Customized'.
                   Default 'CoeffNumDevice'.)DOC")
Y
yuyang18 已提交
809 810 811 812 813
      .def_property(
          "debug_graphviz_path",
          [](const BuildStrategy &self) { return self.debug_graphviz_path_; },
          [](BuildStrategy &self, const std::string &path) {
            self.debug_graphviz_path_ = path;
C
chengduo 已提交
814 815 816 817
          },
          R"DOC(The type is STR, debug_graphviz_path indicate the path that
                    writing the SSA Graph to file in the form of graphviz, you.
                    It is useful for debugging. Default "")DOC")
F
fengjiayi 已提交
818 819 820
      .def_property(
          "enable_data_balance",
          [](const BuildStrategy &self) { return self.enable_data_balance_; },
C
chengduo 已提交
821 822 823 824 825 826 827 828 829 830 831 832 833 834
          [](BuildStrategy &self, bool b) {
            self.enable_data_balance_ = b;
          })  // FIXME(chengudo): enable_data_balance seems not important
      .def_property(
          "fuse_elewise_add_act_ops",
          [](const BuildStrategy &self) {
            return self.fuse_elewise_add_act_ops_;
          },
          [](BuildStrategy &self, bool b) {
            self.fuse_elewise_add_act_ops_ = b;
          },
          R"DOC(The type is BOOL, fuse_elewise_add_act_ops indicate whether
                     to fuse elementwise_add_op and activation_op,
                     it may make the execution faster. Default False)DOC")
835
      .def("_create_passes_from_strategy",
X
fix  
Xin Pan 已提交
836 837 838
           [](BuildStrategy &self) -> std::shared_ptr<ir::PassBuilder> {
             return self.CreatePassesFromStrategy();
           });
Y
yuyang18 已提交
839 840 841 842

  pe.def(py::init<const std::vector<platform::Place> &,
                  const std::unordered_set<std::string> &,
                  const std::unordered_set<std::string> &, const ProgramDesc &,
Y
yuyang18 已提交
843
                  const std::string &, Scope *, std::vector<Scope *> &,
844 845
                  const ExecutionStrategy &, const BuildStrategy &, size_t,
                  size_t>())
Y
Yu Yang 已提交
846 847 848 849
      // NOTE: even we return a vec<Scope*>* to Python use reference policy.
      // We still cannot get local_scope from this vector, since the element
      // of vec<Scope*> will be freed by Python GC. We can only return Scope*
      // one by one and mark them as reference.
850 851 852 853 854
      .def("local_scopes",
           [](ParallelExecutor &self) -> std::vector<Scope *> * {
             return &self.GetLocalScopes();
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
855 856 857 858
      .def("feed_tensors_into_local_scopes",
           &ParallelExecutor::FeedTensorsIntoLocalScopes)
      .def("feed_and_split_tensor_into_local_scopes",
           &ParallelExecutor::FeedAndSplitTensorIntoLocalScopes)
S
sneaxiy 已提交
859 860 861 862 863 864
      .def("run", [](ParallelExecutor &self,
                     const std::vector<std::string> &fetch_tensors,
                     const std::string &fetched_var_name) {
        pybind11::gil_scoped_release release;
        self.Run(fetch_tensors, fetched_var_name);
      });
Y
Yu Yang 已提交
865

866
  BindRecordIOWriter(&m);
867
  return m.ptr();
L
Luo Tao 已提交
868
}
869
}  // namespace pybind
870
}  // namespace paddle