pybind.cc 30.4 KB
Newer Older
1
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
2 3 4 5 6

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

7
http://www.apache.org/licenses/LICENSE-2.0
8 9 10 11 12 13

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
L
lgone2000 已提交
14
#include <Python.h>
C
chengduoZH 已提交
15 16
#include <algorithm>
#include <map>
S
sneaxiy 已提交
17
#include <memory>
C
chengduoZH 已提交
18 19 20 21 22
#include <mutex>  // NOLINT // for call_once
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>
23

24
#include "paddle/fluid/framework/channel.h"
Y
Yi Wang 已提交
25 26 27
#include "paddle/fluid/framework/executor.h"
#include "paddle/fluid/framework/feed_fetch_method.h"
#include "paddle/fluid/framework/framework.pb.h"
28
#include "paddle/fluid/framework/ir/pass_builder.h"
Y
Yi Wang 已提交
29 30 31
#include "paddle/fluid/framework/lod_rank_table.h"
#include "paddle/fluid/framework/lod_tensor.h"
#include "paddle/fluid/framework/lod_tensor_array.h"
32
#include "paddle/fluid/framework/op_registry.h"
Y
Yu Yang 已提交
33
#include "paddle/fluid/framework/parallel_executor.h"
Y
Yi Wang 已提交
34
#include "paddle/fluid/framework/prune.h"
Y
Refine  
Yu Yang 已提交
35
#include "paddle/fluid/framework/reader.h"
Y
Yi Wang 已提交
36
#include "paddle/fluid/framework/selected_rows.h"
X
Xin Pan 已提交
37
#include "paddle/fluid/framework/version.h"
D
dzhwinter 已提交
38
#include "paddle/fluid/operators/activation_op.h"
S
sneaxiy 已提交
39
#include "paddle/fluid/operators/reader/lod_tensor_blocking_queue.h"
Y
Yi Wang 已提交
40
#include "paddle/fluid/platform/enforce.h"
41
#include "paddle/fluid/platform/init.h"
Y
Yi Wang 已提交
42 43 44 45
#include "paddle/fluid/platform/place.h"
#include "paddle/fluid/platform/profiler.h"
#include "paddle/fluid/pybind/const_value.h"
#include "paddle/fluid/pybind/exception.h"
46 47
#include "paddle/fluid/pybind/protobuf.h"
#include "paddle/fluid/pybind/pybind.h"  // NOLINT
Y
Yu Yang 已提交
48
#include "paddle/fluid/pybind/recordio.h"
Y
Yi Wang 已提交
49
#include "paddle/fluid/pybind/tensor_py.h"
Y
Yu Yang 已提交
50

51
#include "paddle/fluid/string/to_string.h"
52

D
Dong Zhihong 已提交
53
#ifdef PADDLE_WITH_CUDA
Y
Yi Wang 已提交
54 55 56
#include "paddle/fluid/operators/nccl/nccl_gpu_common.h"
#include "paddle/fluid/platform/cuda_profiler.h"
#include "paddle/fluid/platform/gpu_info.h"
D
Dong Zhihong 已提交
57 58
#endif

M
minqiyang 已提交
59 60
#include "pybind11/stl.h"

Q
Qiao Longfei 已提交
61 62 63
// disable auto conversion to list in Python
PYBIND11_MAKE_OPAQUE(paddle::framework::LoDTensorArray);

64
namespace paddle {
65
namespace pybind {
66
bool IsCompiledWithCUDA() {
67
#ifndef PADDLE_WITH_CUDA
Q
qijun 已提交
68 69 70 71 72 73
  return false;
#else
  return true;
#endif
}

Y
update  
Yancey1989 已提交
74
bool IsCompiledWithDIST() {
Y
Yancey1989 已提交
75
#ifdef PADDLE_WITH_DISTRIBUTE
Y
update  
Yancey1989 已提交
76 77 78 79 80 81
  return true;
#else
  return false;
#endif
}

82 83
PYBIND11_PLUGIN(core) {
  py::module m("core", "C++ core of PaddlePaddle");
84

85 86 87 88
  // using framework in this function. Since it is inside a function, it will
  // not cause namespace pollution.
  using namespace paddle::framework;  // NOLINT

89
  BindException(&m);
Y
Yu Yang 已提交
90

91 92 93
  py::class_<Tensor>(m, "Tensor", py::buffer_protocol())
      .def_buffer(
          [](Tensor &self) -> py::buffer_info { return CastToPyBuffer(self); })
Y
yuyang18 已提交
94
      .def("_get_dims",
95
           [](const Tensor &self) { return vectorize(self.dims()); })
Y
yuyang18 已提交
96
      .def("_set_dims",
Q
qijun 已提交
97
           [](Tensor &self, const std::vector<int64_t> &dim) {
Y
Yu Yang 已提交
98
             self.Resize(make_ddim(dim));
Y
Yu Yang 已提交
99
           })
Y
yuyang18 已提交
100
      .def("_set_layout",
D
dzhwinter 已提交
101 102 103
           [](Tensor &self, const std::string &layout) {
             self.set_layout(StringToDataLayout(layout));
           })
Y
yuyang18 已提交
104
      .def("_alloc_float",
D
dzhwinter 已提交
105
           [](Tensor &self, paddle::platform::CUDAPlace &place) {
Q
qijun 已提交
106
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
107
           })
Y
yuyang18 已提交
108
      .def("_alloc_float",
Y
Yu Yang 已提交
109
           [](Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
110
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
111
           })
Y
yuyang18 已提交
112
      .def("_alloc_int",
Y
Yu Yang 已提交
113
           [](Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
114
             self.mutable_data<int>(place);
Y
Yu Yang 已提交
115
           })
Y
yuyang18 已提交
116
      .def("_alloc_int",
D
dzhwinter 已提交
117
           [](Tensor &self, paddle::platform::CUDAPlace &place) {
Q
qijun 已提交
118
             self.mutable_data<int>(place);
Q
qijun 已提交
119
           })
Y
yuyang18 已提交
120
      .def("_alloc_int",
C
chengduoZH 已提交
121 122 123
           [](Tensor &self, paddle::platform::CUDAPinnedPlace &place) {
             self.mutable_data<int>(place);
           })
Y
yuyang18 已提交
124
      .def("_alloc_float",
C
chengduoZH 已提交
125 126 127
           [](Tensor &self, paddle::platform::CUDAPinnedPlace &place) {
             self.mutable_data<float>(place);
           })
Y
Yu Yang 已提交
128 129
      .def("set", PyCPUTensorSetFromArray<float>)
      .def("set", PyCPUTensorSetFromArray<int>)
130
      .def("set", PyCPUTensorSetFromArray<double>)
131
      .def("set", PyCPUTensorSetFromArray<int64_t>)
Y
Yu Yang 已提交
132
      .def("set", PyCPUTensorSetFromArray<bool>)
133
      .def("set", PyCPUTensorSetFromArray<uint16_t>)
F
fengjiayi 已提交
134
      .def("set", PyCPUTensorSetFromArray<uint8_t>)
Q
qingqing01 已提交
135
      .def("set", PyCPUTensorSetFromArray<int8_t>)
136
#ifdef PADDLE_WITH_CUDA
Y
Yu Yang 已提交
137 138
      .def("set", PyCUDATensorSetFromArray<float>)
      .def("set", PyCUDATensorSetFromArray<int>)
139
      .def("set", PyCUDATensorSetFromArray<double>)
140
      .def("set", PyCUDATensorSetFromArray<int64_t>)
Y
Yu Yang 已提交
141
      .def("set", PyCUDATensorSetFromArray<bool>)
142
      .def("set", PyCUDATensorSetFromArray<uint16_t>)
F
fengjiayi 已提交
143
      .def("set", PyCUDATensorSetFromArray<uint8_t>)
Q
qingqing01 已提交
144
      .def("set", PyCUDATensorSetFromArray<int8_t>)
C
chengduoZH 已提交
145 146 147 148 149 150
      .def("set", PyCUDAPinnedTensorSetFromArray<float>)
      .def("set", PyCUDAPinnedTensorSetFromArray<int>)
      .def("set", PyCUDAPinnedTensorSetFromArray<double>)
      .def("set", PyCUDAPinnedTensorSetFromArray<int64_t>)
      .def("set", PyCUDAPinnedTensorSetFromArray<bool>)
      .def("set", PyCUDAPinnedTensorSetFromArray<uint16_t>)
F
fengjiayi 已提交
151
      .def("set", PyCUDAPinnedTensorSetFromArray<uint8_t>)
Q
qingqing01 已提交
152
      .def("set", PyCUDAPinnedTensorSetFromArray<int8_t>)
Q
qijun 已提交
153
#endif
154
      .def("shape", [](Tensor &self) { return vectorize(self.dims()); })
Y
yuyang18 已提交
155 156 157 158 159
      .def("_set_float_element", TensorSetElement<float>)
      .def("_get_float_element", TensorGetElement<float>)
      .def("_set_double_element", TensorSetElement<double>)
      .def("_get_double_element", TensorGetElement<double>)
      .def("_dtype", [](Tensor &self) { return ToDataType(self.type()); });
Y
Yu Yang 已提交
160

161
  py::class_<LoDTensor, Tensor>(m, "LoDTensor")
162 163
      .def_buffer(
          [](Tensor &self) -> py::buffer_info { return CastToPyBuffer(self); })
164 165 166 167 168 169 170 171 172 173 174 175 176 177
      .def("__init__",
           [](LoDTensor &instance, const std::vector<std::vector<size_t>>
                                       &recursive_sequence_lengths) {
             LoD new_lod;
             new_lod.reserve(recursive_sequence_lengths.size());
             std::copy(recursive_sequence_lengths.begin(),
                       recursive_sequence_lengths.end(),
                       std::back_inserter(new_lod));
             LoD new_offset_lod = ConvertToOffsetBasedLoD(new_lod);
             PADDLE_ENFORCE(
                 CheckLoD(new_offset_lod, -1),
                 "the provided recursive_sequence_lengths info is invalid");
             new (&instance) LoDTensor(new_offset_lod);
           })
Y
Yu Yang 已提交
178
      .def("__init__", [](LoDTensor &instance) { new (&instance) LoDTensor(); })
G
gongweibao 已提交
179 180 181 182 183
      // We implement offset based LOD in C++ while we use length based with
      // Python API. So we changed set_lod to set_recursive_sequence_lengths to
      // avoid misuse.
      // The discussion is here:
      // https://github.com/PaddlePaddle/Paddle/issues/10855
D
dangqingqing 已提交
184
      .def("set_lod",
185
           [](LoDTensor &self, const std::vector<std::vector<size_t>> &lod) {
186
             // the input lod is offset-based level-of-detail info
Y
Yu Yang 已提交
187
             LoD new_lod;
188 189
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
190 191
             PADDLE_ENFORCE(CheckLoD(new_lod, vectorize(self.dims()).front()),
                            "the provided lod info is invalid");
192
             self.set_lod(new_lod);
D
dangqingqing 已提交
193
           })
194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218
      .def("set_recursive_sequence_lengths",
           [](LoDTensor &self, const std::vector<std::vector<size_t>>
                                   &recursive_sequence_lengths) {
             // the input recursive_sequence_lengths is length-based
             // level-of-detail info
             LoD new_lod;
             new_lod.reserve(recursive_sequence_lengths.size());
             std::copy(recursive_sequence_lengths.begin(),
                       recursive_sequence_lengths.end(),
                       std::back_inserter(new_lod));
             LoD new_offset_lod = ConvertToOffsetBasedLoD(new_lod);
             PADDLE_ENFORCE(
                 CheckLoD(new_offset_lod, vectorize(self.dims()).front()),
                 "the provided recursive_sequence_lengths info is invalid");
             self.set_lod(new_offset_lod);
           })
      .def("lod",
           [](LoDTensor &self) -> std::vector<std::vector<size_t>> {
             // output the offset-based lod info
             LoD lod = self.lod();
             std::vector<std::vector<size_t>> new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             return new_lod;
           })
G
gongweibao 已提交
219
      // Set above comments of set_lod.
220 221 222 223 224 225 226 227 228 229 230 231 232
      .def("recursive_sequence_lengths",
           [](LoDTensor &self) -> std::vector<std::vector<size_t>> {
             // output the length-based lod info
             LoD lod = ConvertToLengthBasedLoD(self.lod());
             std::vector<std::vector<size_t>> new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             return new_lod;
           })
      .def("has_valid_recursive_sequence_lengths", [](LoDTensor &self) -> bool {
        // Check that the lod info is valid and match the outermost
        // dimension of the LoDTensor data
        return CheckLoD(self.lod(), vectorize(self.dims()).front());
D
dangqingqing 已提交
233 234
      });

Q
qijun 已提交
235 236 237 238 239 240 241 242 243 244 245 246 247
  py::class_<SelectedRows>(m, "SelectedRows")
      .def("__init__",
           [](SelectedRows &instance) { new (&instance) SelectedRows(); })
      .def("__init__",
           [](SelectedRows &instance, const std::vector<int64_t> rows,
              const int64_t &height) {
             new (&instance) SelectedRows(rows, height);
           })
      .def("get_tensor",
           [](SelectedRows &self) { return self.mutable_value(); },
           py::return_value_policy::reference)
      .def("set_height", &SelectedRows::set_height)
      .def("height", &SelectedRows::height)
Q
qijun 已提交
248 249 250 251 252 253 254 255 256
      .def("set_rows",
           [](SelectedRows &self, std::vector<int64_t> rows) {
#ifndef PADDLE_WITH_CUDA
             self.set_rows(rows);
#else
        Vector<int64_t> new_rows(rows);
        self.set_rows(new_rows);
#endif
           })
257
      .def("sync_index", [](SelectedRows &instance) { instance.SyncIndex(); })
258
      .def("rows", [](SelectedRows &self) {
259 260 261 262 263
        auto rows = self.rows();
        std::vector<int64_t> new_rows;
        new_rows.reserve(rows.size());
        std::copy(rows.begin(), rows.end(), std::back_inserter(new_rows));
        return new_rows;
264
      });
Q
qijun 已提交
265

266
  py::class_<Variable>(m, "Variable", R"DOC(Variable Class.
267 268 269

All parameter, weight, gradient are variables in Paddle.
)DOC")
270
      .def("is_int", [](const Variable &var) { return var.IsType<int>(); })
271
      .def("set_int",
272 273
           [](Variable &var, int val) -> void { *var.GetMutable<int>() = val; })
      .def("get_int", [](const Variable &var) -> int { return var.Get<int>(); })
274 275 276 277 278 279 280
      .def("is_float", [](const Variable &var) { return var.IsType<float>(); })
      .def("set_float",
           [](Variable &var, float val) -> void {
             *var.GetMutable<float>() = val;
           })
      .def("get_float",
           [](const Variable &var) -> float { return var.Get<float>(); })
Y
Yu Yang 已提交
281
      .def("get_tensor",
282 283
           [](Variable &self) -> LoDTensor * {
             return self.GetMutable<LoDTensor>();
D
dangqingqing 已提交
284 285
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
286 287 288
      .def("get_lod_rank_table",
           [](Variable &self) { return self.GetMutable<LoDRankTable>(); },
           py::return_value_policy::reference)
Q
qijun 已提交
289 290 291 292 293
      .def("get_selected_rows",
           [](Variable &self) -> SelectedRows * {
             return self.GetMutable<SelectedRows>();
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
294 295 296
      .def("get_lod_tensor_array",
           [](Variable &self) { return self.GetMutable<LoDTensorArray>(); },
           py::return_value_policy::reference)
D
Dong Zhihong 已提交
297 298 299 300 301 302 303
#ifdef PADDLE_WITH_CUDA
      .def("get_communicator",
           [](Variable &self) -> platform::Communicator * {
             return self.GetMutable<platform::Communicator>();
           },
           py::return_value_policy::reference)
#endif
Y
Refine  
Yu Yang 已提交
304 305 306 307 308
      .def("get_reader",
           [](Variable &self) -> framework::ReaderHolder * {
             PADDLE_ENFORCE(self.IsType<framework::ReaderHolder>());
             return self.GetMutable<framework::ReaderHolder>();
           },
Y
Yu Yang 已提交
309
           py::return_value_policy::reference);
310

Y
Refine  
Yu Yang 已提交
311
  py::class_<framework::ReaderHolder>(m, "Reader", "")
312
      .def("reset", &framework::ReaderHolder::ResetAll);
Y
Refine  
Yu Yang 已提交
313

S
sneaxiy 已提交
314 315 316 317
  using LoDTensorBlockingQueue =
      ::paddle::operators::reader::LoDTensorBlockingQueue;
  using LoDTensorBlockingQueueHolder =
      ::paddle::operators::reader::LoDTensorBlockingQueueHolder;
S
sneaxiy 已提交
318 319
  py::class_<LoDTensorBlockingQueue, std::shared_ptr<LoDTensorBlockingQueue>>(
      m, "LoDTensorBlockingQueue", "")
S
sneaxiy 已提交
320
      .def("push",
S
sneaxiy 已提交
321
           [](LoDTensorBlockingQueue &self,
S
sneaxiy 已提交
322
              const std::vector<framework::LoDTensor> &lod_tensor_vec) {
S
sneaxiy 已提交
323
             pybind11::gil_scoped_release release;
S
sneaxiy 已提交
324
             return self.Push(lod_tensor_vec);
S
sneaxiy 已提交
325
           })
S
sneaxiy 已提交
326 327 328 329
      .def("size", &LoDTensorBlockingQueue::Size)
      .def("capacity", &LoDTensorBlockingQueue::Cap)
      .def("close", &LoDTensorBlockingQueue::Close)
      .def("is_closed", &LoDTensorBlockingQueue::IsClosed);
S
sneaxiy 已提交
330

S
sneaxiy 已提交
331
  m.def("init_lod_tensor_blocking_queue",
S
sneaxiy 已提交
332
        [](Variable &var, size_t capacity,
S
sneaxiy 已提交
333
           const std::vector<std::vector<int64_t>> &shapes)
S
sneaxiy 已提交
334
            -> std::shared_ptr<LoDTensorBlockingQueue> {
S
sneaxiy 已提交
335 336 337 338 339 340 341
              std::vector<DDim> dims(shapes.size());
              std::transform(shapes.begin(), shapes.end(), dims.begin(),
                             [](const std::vector<int64_t> &shape) {
                               return make_ddim(shape);
                             });
              auto *holder = var.GetMutable<LoDTensorBlockingQueueHolder>();
              holder->InitOnce(capacity, dims);
S
sneaxiy 已提交
342
              return holder->GetQueue();
S
sneaxiy 已提交
343
            },
S
sneaxiy 已提交
344
        py::return_value_policy::copy);
S
sneaxiy 已提交
345

346
  py::class_<Scope>(m, "Scope", "")
D
dongzhihong 已提交
347
      .def("var",
348
           [](Scope &self, const std::string &name) -> Variable * {
D
dongzhihong 已提交
349
             return self.Var(name);
Y
Yu Yang 已提交
350
           },
351
           py::return_value_policy::reference)
352
      .def("find_var", &Scope::FindVar, py::return_value_policy::reference)
Y
Yu Yang 已提交
353
      .def(py::init<>())
354
      .def("new_scope", [](Scope &self) -> Scope * { return &self.NewScope(); },
355
           py::return_value_policy::reference)
Y
Yu Yang 已提交
356
      .def("drop_kids", &Scope::DropKids);
357

Y
Yu Yang 已提交
358 359
  //! @note: Be careful! PyBind will return std::string as an unicode, not
  //! Python str. If you want a str object, you should cast them in Python.
Y
Yu Yang 已提交
360 361
  m.def("get_all_op_protos", []() -> std::vector<py::bytes> {
    std::vector<py::bytes> ret_values;
362 363 364 365 366 367 368 369 370 371
    for (auto &iter : OpInfoMap::Instance().map()) {
      auto &info = iter.second;
      if (info.HasOpProtoAndChecker()) {
        std::string str;
        PADDLE_ENFORCE(
            info.Proto().SerializeToString(&str),
            "Serialize OpProto Error. This could be a bug of Paddle.");
        ret_values.emplace_back(str);
      }
    }
Y
Yu Yang 已提交
372 373
    return ret_values;
  });
374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389
  m.def(
      "get_grad_op_desc", [](const OpDesc &op_desc,
                             const std::unordered_set<std::string> &no_grad_set,
                             const std::vector<BlockDesc *> &grad_sub_block) {
        std::unordered_map<std::string, std::string> grad_to_var;
        std::vector<std::unique_ptr<OpDesc>> grad_op_descs =
            framework::OpInfoMap::Instance()
                .Get(op_desc.Type())
                .GradOpMaker()(op_desc, no_grad_set, &grad_to_var,
                               grad_sub_block);
        std::vector<OpDesc *> grad_op_desc_ptrs(grad_op_descs.size());
        std::transform(grad_op_descs.begin(), grad_op_descs.end(),
                       grad_op_desc_ptrs.begin(),
                       [](std::unique_ptr<OpDesc> &p) { return p.release(); });
        return std::make_pair(grad_op_desc_ptrs, grad_to_var);
      });
Y
Yu Yang 已提交
390
  m.def("prune", [](const ProgramDesc &origin,
391
                    const std::vector<std::array<size_t, 2>> &targets) {
Y
Yu Yang 已提交
392
    ProgramDesc prog_with_targets(origin);
393
    for (const auto &t : targets) {
394
      prog_with_targets.MutableBlock(t[0])->Op(t[1])->SetIsTarget(true);
395
    }
396
    proto::ProgramDesc pruned_desc;
397
    Prune(*prog_with_targets.Proto(), &pruned_desc);
Y
Yu Yang 已提交
398
    return new ProgramDesc(pruned_desc);
399
  });
400 401 402 403
  m.def("empty_var_name",
        []() { return std::string(framework::kEmptyVarName); });
  m.def("grad_var_suffix",
        []() { return std::string(framework::kGradVarSuffix); });
404 405 406
  m.def_submodule(
       "var_names",
       "The module will return special predefined variable name in Paddle")
Y
Yi Wang 已提交
407 408
      .def("empty", []() { return kEmptyVarName; })
      .def("temp", []() { return kTempVarName; });
Q
qijun 已提交
409
  // clang-format off
Y
Yu Yang 已提交
410
  py::class_<paddle::platform::DeviceContext>(m, "DeviceContext")
Q
qijun 已提交
411 412
      .def_static("create",
                  [](paddle::platform::CPUPlace& place)
Q
qijun 已提交
413
                      -> paddle::platform::DeviceContext* {
Q
qijun 已提交
414 415 416
                    return new paddle::platform::CPUDeviceContext();
                  })
      .def_static("create",
D
dzhwinter 已提交
417
                  [](paddle::platform::CUDAPlace& place)
Q
qijun 已提交
418
                      -> paddle::platform::DeviceContext* {
419
#ifndef PADDLE_WITH_CUDA
D
dzhwinter 已提交
420
                    PADDLE_THROW("CUDAPlace is not supported in CPU device.");
Q
qijun 已提交
421
#else
Q
qijun 已提交
422
                    return new paddle::platform::CUDADeviceContext(place);
Q
qijun 已提交
423
#endif
C
chengduoZH 已提交
424 425 426 427 428 429 430 431 432 433 434
                  })
          .def_static("create",
                [](paddle::platform::CUDAPinnedPlace& place)
                        -> paddle::platform::DeviceContext* {
#ifndef PADDLE_WITH_CUDA
                  PADDLE_THROW(
                        "CUDAPinnedPlace is not supported in CPU device.");
#else
                  return new paddle::platform::CUDAPinnedDeviceContext(place);
#endif
                });;
D
Dong Zhihong 已提交
435 436 437 438
// clang-format on
#ifdef PADDLE_WITH_CUDA
  py::class_<platform::Communicator>(m, "Communicator").def(py::init<>());
#endif
D
dzhwinter 已提交
439
  py::class_<platform::CUDAPlace>(m, "CUDAPlace")
440
      .def(py::init<int>())
D
dzhwinter 已提交
441
      .def("__str__", string::to_string<const platform::CUDAPlace &>);
Q
qijun 已提交
442

443 444 445
  py::class_<paddle::platform::CPUPlace>(m, "CPUPlace")
      .def(py::init<>())
      .def("__str__", string::to_string<const platform::CPUPlace &>);
Y
Yu Yang 已提交
446

C
chengduoZH 已提交
447 448 449 450
  py::class_<paddle::platform::CUDAPinnedPlace>(m, "CUDAPinnedPlace")
      .def(py::init<>())
      .def("__str__", string::to_string<const platform::CUDAPinnedPlace &>);

Y
Yu Yang 已提交
451 452 453 454 455 456 457
  py::class_<platform::Place>(m, "Place")
      .def(py::init<>())
      .def("set_place",
           [](platform::Place &self, const platform::CPUPlace &cpu_place) {
             self = cpu_place;
           })
      .def("set_place",
D
dzhwinter 已提交
458
           [](platform::Place &self, const platform::CUDAPlace &gpu_place) {
Y
Yu Yang 已提交
459
             self = gpu_place;
C
chengduoZH 已提交
460 461
           })
      .def("set_place", [](platform::Place &self,
C
chengduoZH 已提交
462 463
                           const platform::CUDAPinnedPlace &cuda_pinned_place) {
        self = cuda_pinned_place;
C
chengduoZH 已提交
464
      });
Y
Yu Yang 已提交
465

Y
Yu Yang 已提交
466 467 468
  py::class_<OperatorBase>(m, "Operator")
      .def_static("create",
                  [](py::bytes protobin) {
469
                    proto::OpDesc desc;
Y
Yu Yang 已提交
470 471 472 473 474
                    PADDLE_ENFORCE(desc.ParsePartialFromString(protobin),
                                   "Cannot parse user input to OpDesc");
                    PADDLE_ENFORCE(desc.IsInitialized(),
                                   "User OpDesc is not initialized, reason %s",
                                   desc.InitializationErrorString());
475
                    return OpRegistry::CreateOp(desc);
Y
Yu Yang 已提交
476
                  })
477
      .def("run",
478
           [](OperatorBase &self, const Scope &scope,
D
dzhwinter 已提交
479 480 481
              const platform::CPUPlace &place) { self.Run(scope, place); })
      .def("run",
           [](OperatorBase &self, const Scope &scope,
D
dzhwinter 已提交
482
              const platform::CUDAPlace &place) { self.Run(scope, place); })
C
chengduoZH 已提交
483 484 485 486 487
      .def("run",
           [](OperatorBase &self, const Scope &scope,
              const platform::CUDAPinnedPlace &place) {
             self.Run(scope, place);
           })
Y
Yu Yang 已提交
488 489 490 491 492 493 494
      .def("type",
           [](const OperatorBase &op) -> std::string { return op.Type(); })
      .def("outputs",
           [](const OperatorBase &op)
               -> std::map<std::string, std::vector<std::string>> {
                 return op.Outputs();
               })
Q
qijun 已提交
495 496
      .def("output_vars",
           [](const OperatorBase &op) { return op.OutputVars(true); })
Y
Yu Yang 已提交
497
      .def("inputs", [](const OperatorBase &op) { return op.Inputs(); })
Q
qijun 已提交
498
      .def("input_vars", [](const OperatorBase &op) { return op.InputVars(); })
Y
Yu Yang 已提交
499 500 501 502
      .def("__str__", &OperatorBase::DebugString)
      .def("no_intermediate_outputs",
           [](const OperatorBase &op) { return op.OutputVars(false); })
      .def("support_gpu", &OperatorBase::SupportGPU);
Y
Yu Yang 已提交
503

F
fengjiayi 已提交
504
  py::class_<framework::Executor>(m, "Executor")
D
dzhwinter 已提交
505
      .def(py::init<const platform::Place &>())
Y
Yancey1989 已提交
506
      .def("close", &Executor::Close)
S
sneaxiy 已提交
507 508 509 510 511
      .def("run", [](Executor &self, const ProgramDesc &prog, Scope *scope,
                     int block_id, bool create_local_scope, bool create_vars) {
        pybind11::gil_scoped_release release;
        self.Run(prog, scope, block_id, create_local_scope, create_vars);
      });
S
sneaxiy 已提交
512

D
dzhwinter 已提交
513
  m.def("init_gflags", framework::InitGflags);
Y
Yang Yu 已提交
514
  m.def("init_glog", framework::InitGLOG);
X
Xin Pan 已提交
515 516
  m.def("init_devices",
        [](bool init_p2p) { framework::InitDevices(init_p2p); });
517

518
  m.def("is_compiled_with_cuda", IsCompiledWithCUDA);
Y
update  
Yancey1989 已提交
519
  m.def("is_compiled_with_dist", IsCompiledWithDIST);
520 521 522 523 524 525
#ifdef PADDLE_WITH_CUDA
  m.def("is_float16_supported", [](const platform::CUDAPlace &place) -> bool {
    // Only GPUs with Compute Capability >= 53 support float16
    return platform::GetCUDAComputeCapability(place.device) >= 53;
  });
#endif
526

527
  m.def("set_feed_variable", framework::SetFeedVariable);
Q
qijun 已提交
528
  m.def("get_fetch_variable", framework::GetFetchVariable);
Q
qijun 已提交
529

X
Xin Pan 已提交
530 531
  m.def("_is_program_version_supported", IsProgramVersionSupported);

532 533 534 535 536
  BindProgramDesc(&m);
  BindBlockDesc(&m);
  BindVarDsec(&m);
  BindOpDesc(&m);
  BindConstValue(&m);
Y
Yu Yang 已提交
537

Y
Yu Yang 已提交
538 539 540 541 542 543 544 545 546
  py::class_<framework::LoDRankTable>(m, "LodRankTable")
      .def("items", [](framework::LoDRankTable &table) {
        std::vector<std::pair<size_t, size_t>> res;
        for (auto &item : table.items()) {
          res.push_back({item.index, item.length});
        }
        return res;
      });

Y
Yu Yang 已提交
547
  py::class_<LoDTensorArray>(m, "LoDTensorArray")
S
sneaxiy 已提交
548 549
      .def("__init__",
           [](LoDTensorArray &instance) { new (&instance) LoDTensorArray(); })
Y
Yu Yang 已提交
550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565
      .def("__getitem__",
           [](LoDTensorArray &self, size_t i) { return &self.at(i); },
           py::return_value_policy::reference)
      .def("__len__", [](LoDTensorArray &self) { return self.size(); })
      .def("__setitem__",
           [](LoDTensorArray &self, size_t i, const LoDTensor &t) {
             PADDLE_ENFORCE_LT(i, self.size());
             self[i].ShareDataWith(t);
             self[i].set_lod(t.lod());
           })
      .def("append", [](LoDTensorArray &self, const LoDTensor &t) {
        self.emplace_back();
        self.back().ShareDataWith(t);
        self.back().set_lod(t.lod());
      });

D
dzhwinter 已提交
566 567 568
  m.def("IsInplace",
        [](std::string op) -> bool { return operators::IsInplace(op); });

Y
Yu Yang 已提交
569
  m.def("op_support_gpu", OpSupportGPU);
D
Dong Zhihong 已提交
570
#ifdef PADDLE_WITH_CUDA
D
Dong Zhihong 已提交
571
  m.def("get_cuda_device_count", platform::GetCUDADeviceCount);
D
dangqingqing 已提交
572 573 574 575

  m.def("nvprof_init", platform::CudaProfilerInit);
  m.def("nvprof_start", platform::CudaProfilerStart);
  m.def("nvprof_stop", platform::CudaProfilerStop);
D
Dong Zhihong 已提交
576
#endif
Y
Yu Yang 已提交
577

578 579 580 581
  py::enum_<platform::ProfilerState>(m, "ProfilerState", py::arithmetic())
      .value("kDisabled", platform::ProfilerState::kDisabled)
      .value("kCPU", platform::ProfilerState::kCPU)
      .value("kCUDA", platform::ProfilerState::kCUDA)
582
      .value("kAll", platform::ProfilerState::kAll)
583 584 585 586 587 588 589 590 591 592 593 594 595
      .export_values();

  py::enum_<platform::EventSortingKey>(m, "EventSortingKey", py::arithmetic())
      .value("kDefault", platform::EventSortingKey::kDefault)
      .value("kCalls", platform::EventSortingKey::kCalls)
      .value("kTotal", platform::EventSortingKey::kTotal)
      .value("kMin", platform::EventSortingKey::kMin)
      .value("kMax", platform::EventSortingKey::kMax)
      .value("kAve", platform::EventSortingKey::kAve)
      .export_values();

  m.def("enable_profiler", platform::EnableProfiler);
  m.def("disable_profiler", platform::DisableProfiler);
X
Xin Pan 已提交
596
  m.def("is_profiler_enabled", platform::IsProfileEnabled);
597
  m.def("reset_profiler", platform::ResetProfiler);
Y
Yu Yang 已提交
598

599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620
  py::class_<ir::Pass, std::shared_ptr<ir::Pass>> pass(m, "Pass");
  pass.def(py::init())
      .def("set_str", [](ir::Pass &self, const std::string &name,
                         const std::string &attr) {
        self.Set<std::string>(name, new std::string(attr));
      });

  py::class_<ir::PassBuilder> pb(m, "PassBuilder");
  pb.def(py::init())
      .def("append_pass",
           [](ir::PassBuilder &self,
              const std::string &pass_type) -> std::shared_ptr<ir::Pass> {
             return self.AppendPass(pass_type);
           })
      .def("all_passes", [](ir::PassBuilder &self) { return self.AllPasses(); })
      .def("insert_pass",
           [](ir::PassBuilder &self, size_t idx, const std::string &pass_type) {
             return self.InsertPass(idx, pass_type);
           })
      .def("remove_pass",
           [](ir::PassBuilder &self, size_t idx) { self.RemovePass(idx); });

Y
yuyang18 已提交
621
  // -- python binds for parallel executor.
Y
yuyang18 已提交
622
  py::class_<ParallelExecutor> pe(m, "ParallelExecutor");
Y
yuyang18 已提交
623 624
  py::class_<ExecutionStrategy> exec_strategy(pe, "ExecutionStrategy");
  exec_strategy.def(py::init())
Y
yuyang18 已提交
625 626 627 628 629 630 631
      .def_property(
          "num_threads",
          [](const ExecutionStrategy &self) { return self.num_threads_; },
          [](ExecutionStrategy &self, size_t num_threads) {
            self.num_threads_ = num_threads;
          })
      .def_property(
632 633 634 635
          "use_cuda",
          [](const ExecutionStrategy &self) { return self.use_cuda_; },
          [](ExecutionStrategy &self, bool use_cuda) {
            self.use_cuda_ = use_cuda;
Y
yuyang18 已提交
636 637 638 639 640 641
          })
      .def_property(
          "allow_op_delay",
          [](const ExecutionStrategy &self) { return self.allow_op_delay_; },
          [](ExecutionStrategy &self, bool allow_op_delay) {
            self.allow_op_delay_ = allow_op_delay;
Y
yuyang18 已提交
642 643 644 645 646 647 648 649
          })
      .def_property(
          "num_iteration_per_drop_scope",
          [](const ExecutionStrategy &self) {
            return self.num_iteration_per_drop_scope_;
          },
          [](ExecutionStrategy &self, size_t num_iteration_per_drop_scope) {
            self.num_iteration_per_drop_scope_ = num_iteration_per_drop_scope;
Y
yuyang18 已提交
650
          });
Y
yuyang18 已提交
651
  exec_strategy.def_property(
Y
yuyang18 已提交
652 653 654 655 656 657 658
      "use_experimental_executor",
      [](const ExecutionStrategy &self) {
        return self.type_ == ExecutionStrategy::kExperimental;
      },
      [](ExecutionStrategy &self, bool experimental) {
        self.type_ = experimental ? ExecutionStrategy::kExperimental
                                  : ExecutionStrategy::kDefault;
Y
yuyang18 已提交
659 660
      });

Y
yuyang18 已提交
661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685
  py::class_<BuildStrategy> build_strategy(pe, "BuildStrategy");

  py::enum_<BuildStrategy::ReduceStrategy>(build_strategy, "ReduceStrategy")
      .value("Reduce", BuildStrategy::ReduceStrategy::kReduce)
      .value("AllReduce", BuildStrategy::ReduceStrategy::kAllReduce);
  py::enum_<BuildStrategy::GradientScaleStrategy>(build_strategy,
                                                  "GradientScaleStrategy")
      .value("CoeffNumDevice",
             BuildStrategy::GradientScaleStrategy::kCoeffNumDevice)
      .value("One", BuildStrategy::GradientScaleStrategy::kOne)
      .value("Customized", BuildStrategy::GradientScaleStrategy::kCustomized);

  build_strategy.def(py::init())
      .def_property(
          "reduce_strategy",
          [](const BuildStrategy &self) { return self.reduce_; },
          [](BuildStrategy &self, BuildStrategy::ReduceStrategy strategy) {
            self.reduce_ = strategy;
          })
      .def_property(
          "gradient_scale_strategy",
          [](const BuildStrategy &self) { return self.gradient_scale_; },
          [](BuildStrategy &self,
             BuildStrategy::GradientScaleStrategy strategy) {
            self.gradient_scale_ = strategy;
Y
yuyang18 已提交
686 687 688 689 690 691
          })
      .def_property(
          "debug_graphviz_path",
          [](const BuildStrategy &self) { return self.debug_graphviz_path_; },
          [](BuildStrategy &self, const std::string &path) {
            self.debug_graphviz_path_ = path;
F
fengjiayi 已提交
692 693 694 695
          })
      .def_property(
          "enable_data_balance",
          [](const BuildStrategy &self) { return self.enable_data_balance_; },
C
chengduo 已提交
696 697 698 699 700 701 702
          [](BuildStrategy &self, bool b) { self.enable_data_balance_ = b; })
      .def_property("fuse_elewise_add_act_ops",
                    [](const BuildStrategy &self) {
                      return self.fuse_elewise_add_act_ops_;
                    },
                    [](BuildStrategy &self, bool b) {
                      self.fuse_elewise_add_act_ops_ = b;
703 704 705 706
                    })
      .def("create_pass_builder",
           [](BuildStrategy &self) { return *self.CreatePassBuilder(); },
           py::return_value_policy::reference);
Y
yuyang18 已提交
707 708 709 710

  pe.def(py::init<const std::vector<platform::Place> &,
                  const std::unordered_set<std::string> &,
                  const std::unordered_set<std::string> &, const ProgramDesc &,
Y
yuyang18 已提交
711
                  const std::string &, Scope *, std::vector<Scope *> &,
712 713
                  const ExecutionStrategy &, const BuildStrategy &, size_t,
                  size_t>())
Y
Yu Yang 已提交
714 715 716 717
      // NOTE: even we return a vec<Scope*>* to Python use reference policy.
      // We still cannot get local_scope from this vector, since the element
      // of vec<Scope*> will be freed by Python GC. We can only return Scope*
      // one by one and mark them as reference.
718 719 720 721 722
      .def("local_scopes",
           [](ParallelExecutor &self) -> std::vector<Scope *> * {
             return &self.GetLocalScopes();
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
723 724 725 726
      .def("feed_tensors_into_local_scopes",
           &ParallelExecutor::FeedTensorsIntoLocalScopes)
      .def("feed_and_split_tensor_into_local_scopes",
           &ParallelExecutor::FeedAndSplitTensorIntoLocalScopes)
S
sneaxiy 已提交
727 728 729 730 731 732
      .def("run", [](ParallelExecutor &self,
                     const std::vector<std::string> &fetch_tensors,
                     const std::string &fetched_var_name) {
        pybind11::gil_scoped_release release;
        self.Run(fetch_tensors, fetched_var_name);
      });
Y
Yu Yang 已提交
733

734
  BindRecordIOWriter(&m);
735
  return m.ptr();
L
Luo Tao 已提交
736
}
737
}  // namespace pybind
738
}  // namespace paddle