pybind.cc 14.6 KB
Newer Older
1 2 3 4 5 6
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

7
http://www.apache.org/licenses/LICENSE-2.0
8 9 10 11 12 13 14

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

15
#include "paddle/pybind/protobuf.h"
16

Q
Qiao Longfei 已提交
17
#include "paddle/framework/backward.h"
D
dangqingqing 已提交
18
#include "paddle/framework/lod_tensor.h"
19
#include "paddle/framework/tensor_array.h"
Z
zchen0211 已提交
20
#include "paddle/operators/cond_op.h"
Y
Yan Chunwei 已提交
21
#include "paddle/operators/net_op.h"
Y
Yan Chunwei 已提交
22
#include "paddle/operators/recurrent_op.h"
Q
qijun 已提交
23
#include "paddle/platform/enforce.h"
Q
qijun 已提交
24
#include "paddle/platform/place.h"
Y
Yu Yang 已提交
25
#include "paddle/pybind/exception.h"
L
Luo Tao 已提交
26
#include "paddle/pybind/pybind.h"
27
#include "paddle/pybind/tensor_py.h"
28
#include "paddle/string/to_string.h"
29

30
namespace paddle {
31
namespace pybind {
32 33 34 35 36
static size_t UniqueIntegerGenerator() {
  static std::atomic<size_t> generator;
  return generator.fetch_add(1);
}

Q
qijun 已提交
37
bool IsCompileGPU() {
38
#ifndef PADDLE_WITH_CUDA
Q
qijun 已提交
39 40 41 42 43 44
  return false;
#else
  return true;
#endif
}

45
PYBIND11_PLUGIN(core) {
Y
Yu Yang 已提交
46
  py::module m("core", "C++ core of PaddlePaddle");
47

48 49 50 51
  // using framework in this function. Since it is inside a function, it will
  // not cause namespace pollution.
  using namespace paddle::framework;  // NOLINT

Y
Yu Yang 已提交
52 53
  BindException(m);

54 55 56
  py::class_<Tensor>(m, "Tensor", py::buffer_protocol())
      .def_buffer(
          [](Tensor &self) -> py::buffer_info { return CastToPyBuffer(self); })
Y
Yu Yang 已提交
57
      .def("get_dims",
58
           [](const Tensor &self) { return vectorize(self.dims()); })
Y
Yu Yang 已提交
59
      .def("set_dims",
Q
qijun 已提交
60
           [](Tensor &self, const std::vector<int64_t> &dim) {
Y
Yu Yang 已提交
61
             self.Resize(make_ddim(dim));
Y
Yu Yang 已提交
62 63
           })
      .def("alloc_float",
Y
Yu Yang 已提交
64
           [](Tensor &self, paddle::platform::GPUPlace &place) {
Q
qijun 已提交
65
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
66
           })
Q
qijun 已提交
67
      .def("alloc_float",
Y
Yu Yang 已提交
68
           [](Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
69
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
70 71
           })
      .def("alloc_int",
Y
Yu Yang 已提交
72
           [](Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
73
             self.mutable_data<int>(place);
Y
Yu Yang 已提交
74
           })
Q
qijun 已提交
75
      .def("alloc_int",
Y
Yu Yang 已提交
76
           [](Tensor &self, paddle::platform::GPUPlace &place) {
Q
qijun 已提交
77
             self.mutable_data<int>(place);
Q
qijun 已提交
78
           })
Y
Yu Yang 已提交
79 80
      .def("set", PyCPUTensorSetFromArray<float>)
      .def("set", PyCPUTensorSetFromArray<int>)
81
      .def("set", PyCPUTensorSetFromArray<double>)
82
#ifdef PADDLE_WITH_CUDA
Y
Yu Yang 已提交
83 84
      .def("set", PyCUDATensorSetFromArray<float>)
      .def("set", PyCUDATensorSetFromArray<int>)
85
      .def("set", PyCUDATensorSetFromArray<double>)
Q
qijun 已提交
86
#endif
87
      .def("shape", [](Tensor &self) { return vectorize(self.dims()); })
88 89 90 91 92
      .def("set_float_element", TensorSetElement<float>)
      .def("get_float_element", TensorGetElement<float>)
      .def("set_double_element", TensorSetElement<double>)
      .def("get_double_element", TensorGetElement<double>)
      .def("dtype", [](Tensor &self) { return ToDataType(self.type()); });
Y
Yu Yang 已提交
93

94
  py::class_<LoDTensor, Tensor>(m, "LoDTensor")
95 96
      .def_buffer(
          [](Tensor &self) -> py::buffer_info { return CastToPyBuffer(self); })
97 98 99
      .def(
          "__init__",
          [](LoDTensor &instance, const std::vector<std::vector<size_t>> &lod) {
100
#ifndef PADDLE_WITH_CUDA
101
            new (&instance) LoDTensor(lod);
102
#else
Y
Yu Yang 已提交
103
             LoD new_lod;
104 105
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
106
             new (&instance) LoDTensor(new_lod);
107
#endif
108
          })
D
dangqingqing 已提交
109
      .def("set_lod",
110
           [](LoDTensor &self, const std::vector<std::vector<size_t>> &lod) {
111
#ifndef PADDLE_WITH_CUDA
D
dangqingqing 已提交
112
             self.set_lod(lod);
113
#else
Y
Yu Yang 已提交
114
             LoD new_lod;
115 116 117 118
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             self.set_lod(new_lod);
#endif
D
dangqingqing 已提交
119
           })
120
      .def("lod", [](LoDTensor &self) -> std::vector<std::vector<size_t>> {
121
#ifndef PADDLE_WITH_CUDA
D
dangqingqing 已提交
122
        return self.lod();
123 124 125 126 127
#else
           auto lod = self.lod();
           std::vector<std::vector<size_t>> new_lod;
           new_lod.reserve(lod.size());
           std::transform(lod.begin(), lod.end(), std::back_inserter(new_lod),
Y
Yu Yang 已提交
128
               [](Vector<size_t> item) ->
129 130 131 132 133 134 135 136
                   std::vector<size_t> {
                 std::vector<size_t> v;
                 v.reserve(item.size());
                 std::copy(item.begin(), item.end(), std::back_inserter(v));
                 return v;
               });
           return new_lod;
#endif
D
dangqingqing 已提交
137 138
      });

139
  py::class_<Variable>(m, "Variable", R"DOC(Variable Class.
140 141 142

All parameter, weight, gradient are variables in Paddle.
)DOC")
143
      .def("is_int", [](const Variable &var) { return var.IsType<int>(); })
144
      .def("set_int",
145 146
           [](Variable &var, int val) -> void { *var.GetMutable<int>() = val; })
      .def("get_int", [](const Variable &var) -> int { return var.Get<int>(); })
147 148 149 150 151 152 153
      .def("is_float", [](const Variable &var) { return var.IsType<float>(); })
      .def("set_float",
           [](Variable &var, float val) -> void {
             *var.GetMutable<float>() = val;
           })
      .def("get_float",
           [](const Variable &var) -> float { return var.Get<float>(); })
Y
Yu Yang 已提交
154
      .def("get_tensor",
155 156
           [](Variable &self) -> LoDTensor * {
             return self.GetMutable<LoDTensor>();
D
dangqingqing 已提交
157 158
           },
           py::return_value_policy::reference)
Y
Yan Chunwei 已提交
159
      .def("get_net",
D
dongzhihong 已提交
160 161
           [](Variable &self) -> operators::NetOp * {
             return self.GetMutable<operators::NetOp>();
Y
Yan Chunwei 已提交
162
           },
Y
Yu Yang 已提交
163
           py::return_value_policy::reference);
164

165
  py::class_<Scope>(m, "Scope", "")
Y
Yu Yang 已提交
166
      .def("new_var",
167
           [](Scope &self, const std::string &name) -> Variable * {
Y
Yu Yang 已提交
168 169
             return self.NewVar(name);
           },
170
           py::return_value_policy::reference)
171
      .def("find_var", &Scope::FindVar, py::return_value_policy::reference)
Y
Yu Yang 已提交
172
      .def(py::init<>())
173
      .def("new_scope", [](Scope &self) -> Scope * { return &self.NewScope(); },
174
           py::return_value_policy::reference)
175
      .def("drop_kids", &Scope::DropKids);
176

Y
Yu Yang 已提交
177 178
  //! @note: Be careful! PyBind will return std::string as an unicode, not
  //! Python str. If you want a str object, you should cast them in Python.
Y
Yu Yang 已提交
179 180
  m.def("get_all_op_protos", []() -> std::vector<py::bytes> {
    std::vector<py::bytes> ret_values;
Y
Yu Yang 已提交
181 182 183 184

    OpInfoMap::Instance().IterAllInfo([&ret_values](const std::string &type,
                                                    const OpInfo &info) {
      if (!info.HasOpProtoAndChecker()) return;
Y
Yu Yang 已提交
185
      std::string str;
Y
Yu Yang 已提交
186
      PADDLE_ENFORCE(info.Proto().SerializeToString(&str),
Y
Yu Yang 已提交
187
                     "Serialize OpProto Error. This could be a bug of Paddle.");
Y
Yu Yang 已提交
188 189
      ret_values.emplace_back(str);
    });
Y
Yu Yang 已提交
190 191
    return ret_values;
  });
192 193 194
  m.def_submodule(
       "var_names",
       "The module will return special predefined variable name in Paddle")
Y
Yi Wang 已提交
195 196
      .def("empty", []() { return kEmptyVarName; })
      .def("temp", []() { return kTempVarName; });
Q
qijun 已提交
197
  // clang-format off
Y
Yu Yang 已提交
198
  py::class_<paddle::platform::DeviceContext>(m, "DeviceContext")
Q
qijun 已提交
199 200
      .def_static("create",
                  [](paddle::platform::CPUPlace& place)
Q
qijun 已提交
201
                      -> paddle::platform::DeviceContext* {
Q
qijun 已提交
202 203 204 205 206
                    return new paddle::platform::CPUDeviceContext();
                  })
      .def_static("create",
                  [](paddle::platform::GPUPlace& place)
                      -> paddle::platform::DeviceContext* {
207
#ifndef PADDLE_WITH_CUDA
Q
qijun 已提交
208
                    PADDLE_THROW("GPUPlace is not supported in CPU device.");
Q
qijun 已提交
209
#else
Q
qijun 已提交
210
                    return new paddle::platform::CUDADeviceContext(place);
Q
qijun 已提交
211
#endif
Q
qijun 已提交
212
                  });
Q
qijun 已提交
213
  // clang-format on
Q
qijun 已提交
214

215 216 217
  py::class_<platform::GPUPlace>(m, "GPUPlace")
      .def(py::init<int>())
      .def("__str__", string::to_string<const platform::GPUPlace &>);
Q
qijun 已提交
218

219 220 221
  py::class_<paddle::platform::CPUPlace>(m, "CPUPlace")
      .def(py::init<>())
      .def("__str__", string::to_string<const platform::CPUPlace &>);
Y
Yu Yang 已提交
222

Y
Yu Yang 已提交
223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238
  py::class_<OperatorBase>(m, "Operator")
      .def_static("create",
                  [](py::bytes protobin) {
                    OpDesc desc;
                    PADDLE_ENFORCE(desc.ParsePartialFromString(protobin),
                                   "Cannot parse user input to OpDesc");
                    PADDLE_ENFORCE(desc.IsInitialized(),
                                   "User OpDesc is not initialized, reason %s",
                                   desc.InitializationErrorString());
                    return OpRegistry::CreateOp(desc);
                  })
      .def("backward",
           [](const OperatorBase &forwardOp,
              const std::unordered_set<std::string> &no_grad_vars) {
             return Backward(forwardOp, no_grad_vars).release();
           })
239
      .def("run",
240
           [](OperatorBase &self, const Scope &scope,
241 242 243 244
              const platform::DeviceContext &dev_ctx) {
             self.Run(scope, dev_ctx);
             dev_ctx.Wait();
           })
Y
Yu Yang 已提交
245 246 247 248 249 250 251
      .def("type",
           [](const OperatorBase &op) -> std::string { return op.Type(); })
      .def("outputs",
           [](const OperatorBase &op)
               -> std::map<std::string, std::vector<std::string>> {
                 return op.Outputs();
               })
Q
qijun 已提交
252 253
      .def("output_vars",
           [](const OperatorBase &op) { return op.OutputVars(true); })
Y
Yu Yang 已提交
254
      .def("inputs", [](const OperatorBase &op) { return op.Inputs(); })
Q
qijun 已提交
255
      .def("input_vars", [](const OperatorBase &op) { return op.InputVars(); })
Y
Yu Yang 已提交
256 257 258 259
      .def("__str__", &OperatorBase::DebugString)
      .def("no_intermediate_outputs",
           [](const OperatorBase &op) { return op.OutputVars(false); })
      .def("support_gpu", &OperatorBase::SupportGPU);
Y
Yu Yang 已提交
260

Y
Yu Yang 已提交
261 262 263 264 265 266 267
  py::class_<operators::NetOp, OperatorBase>(m, "Net")
      .def_static("create",
                  []() -> operators::NetOp * {
                    auto *retv = new operators::NetOp;
                    retv->SetType("plain_net");
                    return retv;
                  })
268 269
      .def("append_op", [](operators::NetOp &self,
                           const OperatorBase &op) { self.AppendOp(op); })
D
dongzhihong 已提交
270 271 272 273
      .def("complete_add_op", &operators::NetOp::CompleteAddOp)
      .def("complete_add_op", [](std::shared_ptr<operators::NetOp> &self) {
        self->CompleteAddOp();
      });
Y
Yan Chunwei 已提交
274

275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324
  py::class_<framework::TensorArray>(m, "TensorArray")
      .def("__init__",
           [](TensorArray &instance) { new (&instance) TensorArray(); })
      .def("read",
           [](TensorArray &self, size_t index) { return self.Read(index); })
      .def("write", [](TensorArray &self, size_t index,
                       LoDTensor &value) { self.Write(index, value); })
      .def("write_shared",
           [](TensorArray &self, size_t index, const LoDTensor &value) {
             self.WriteShared(index, value);
           })
      .def("size", [](TensorArray &self) { return self.size(); })
      .def("pack",
           [](TensorArray &self, size_t level,
              const std::vector<std::vector<size_t>> &meta_info,
              const std::vector<std::vector<size_t>> &lod) {
             std::vector<DySeqMeta> meta;
             for (auto &info : meta_info) {
               PADDLE_ENFORCE_EQ(info.size(), 3UL);
               meta.emplace_back(info[0], info[1], info[2]);
             }
#ifndef PADDLE_WITH_CUDA
             return self.Pack(level, meta, lod);
#else
             LoD new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             return self.Pack(level, meta, new_lod);
#endif
           })
      .def("unpack",
           [](TensorArray &self, const LoDTensor &source, int level,
              bool length_descend) {
             auto metas = self.Unpack(source, level, length_descend);
             std::vector<std::vector<size_t>> meta_info;
             for (auto meta : metas) {
               meta_info.emplace_back(
                   std::vector<size_t>({meta.begin, meta.end, meta.ori_idx}));
             }
             return meta_info;
           })
      .def("stack", [](TensorArray &self) { return self.Stack(); })
      .def("unstack",
           [](TensorArray &self, const LoDTensor &source) {
             return self.Unstack(source);
           })
      .def("unstack_shared", [](TensorArray &self, const LoDTensor &source) {
        return self.UnstackShared(source);
      });

Y
Yan Chunwei 已提交
325
  // recurrent_op
Y
Yu Yang 已提交
326 327 328 329 330 331 332 333 334 335 336 337 338
  py::class_<operators::RecurrentOp, OperatorBase>(m, "RecurrentOp")
      .def_static(
          "create",
          [](py::bytes protobin) -> operators::RecurrentOp * {
            OpDesc desc;
            PADDLE_ENFORCE(desc.ParsePartialFromString(protobin),
                           "Cannot parse user input to OpDesc");
            PADDLE_ENFORCE(desc.IsInitialized(),
                           "User OpDesc is not initialized, reason %s",
                           desc.InitializationErrorString());
            auto rnn_op = OpRegistry::CreateOp(desc);
            return static_cast<operators::RecurrentOp *>(rnn_op.release());
          })
339 340 341 342
      .def("set_stepnet", [](operators::RecurrentOp &self,
                             const operators::NetOp &net) -> void {
        self.set_stepnet(net.Clone());
      });
Y
Yan Chunwei 已提交
343

Z
cond op  
zchen0211 已提交
344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365
  // cond_op
  py::class_<operators::CondOp, OperatorBase>(m, "CondOp")
      .def_static("create",
                  [](py::bytes protobin) -> operators::CondOp * {
                    OpDesc desc;
                    PADDLE_ENFORCE(desc.ParsePartialFromString(protobin),
                                   "Cannot parse user input to OpDesc");
                    PADDLE_ENFORCE(desc.IsInitialized(),
                                   "User OpDesc is not initialized, reason %s",
                                   desc.InitializationErrorString());
                    auto cond_op = OpRegistry::CreateOp(desc);
                    return static_cast<operators::CondOp *>(cond_op.release());
                  })
      .def("set_truenet",
           [](operators::CondOp &self, const operators::NetOp &net) -> void {
             self.set_truenet(net.Clone());
           })
      .def("set_falsenet",
           [](operators::CondOp &self, const operators::NetOp &net) -> void {
             self.set_falsenet(net.Clone());
           });

366 367
  m.def("unique_integer", UniqueIntegerGenerator);

Q
qijun 已提交
368 369
  m.def("is_compile_gpu", IsCompileGPU);

F
fengjiayi 已提交
370 371 372 373
  BindProgramDesc(m);
  BindBlockDesc(m);
  BindVarDsec(m);
  BindOpDesc(m);
Y
Yu Yang 已提交
374

375
  return m.ptr();
L
Luo Tao 已提交
376
}
377
}  // namespace pybind
378
}  // namespace paddle