pybind.cc 12.5 KB
Newer Older
1 2 3 4 5 6
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

7
http://www.apache.org/licenses/LICENSE-2.0
8 9 10 11 12 13 14

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

15
#include "paddle/pybind/protobuf.h"
16

Q
Qiao Longfei 已提交
17
#include "paddle/framework/backward.h"
D
dangqingqing 已提交
18
#include "paddle/framework/lod_tensor.h"
Z
zchen0211 已提交
19
#include "paddle/operators/cond_op.h"
Y
Yan Chunwei 已提交
20
#include "paddle/operators/net_op.h"
Y
Yan Chunwei 已提交
21
#include "paddle/operators/recurrent_op.h"
Q
qijun 已提交
22
#include "paddle/platform/enforce.h"
Q
qijun 已提交
23
#include "paddle/platform/place.h"
Y
Yu Yang 已提交
24
#include "paddle/pybind/exception.h"
L
Luo Tao 已提交
25
#include "paddle/pybind/pybind.h"
26
#include "paddle/pybind/tensor_py.h"
27
#include "paddle/string/to_string.h"
28

29
namespace paddle {
30
namespace pybind {
31 32 33 34 35
static size_t UniqueIntegerGenerator() {
  static std::atomic<size_t> generator;
  return generator.fetch_add(1);
}

Q
qijun 已提交
36 37 38 39 40 41 42 43
bool IsCompileGPU() {
#ifdef PADDLE_ONLY_CPU
  return false;
#else
  return true;
#endif
}

44
PYBIND11_PLUGIN(core) {
Y
Yu Yang 已提交
45
  py::module m("core", "C++ core of PaddlePaddle");
46

47 48 49 50
  // using framework in this function. Since it is inside a function, it will
  // not cause namespace pollution.
  using namespace paddle::framework;  // NOLINT

Y
Yu Yang 已提交
51 52
  BindException(m);

53 54 55
  py::class_<Tensor>(m, "Tensor", py::buffer_protocol())
      .def_buffer(
          [](Tensor &self) -> py::buffer_info { return CastToPyBuffer(self); })
Y
Yu Yang 已提交
56
      .def("get_dims",
57
           [](const Tensor &self) { return vectorize(self.dims()); })
Y
Yu Yang 已提交
58
      .def("set_dims",
Q
qijun 已提交
59
           [](Tensor &self, const std::vector<int64_t> &dim) {
Y
Yu Yang 已提交
60
             self.Resize(make_ddim(dim));
Y
Yu Yang 已提交
61 62
           })
      .def("alloc_float",
Y
Yu Yang 已提交
63
           [](Tensor &self, paddle::platform::GPUPlace &place) {
Q
qijun 已提交
64
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
65
           })
Q
qijun 已提交
66
      .def("alloc_float",
Y
Yu Yang 已提交
67
           [](Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
68
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
69 70
           })
      .def("alloc_int",
Y
Yu Yang 已提交
71
           [](Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
72
             self.mutable_data<int>(place);
Y
Yu Yang 已提交
73
           })
Q
qijun 已提交
74
      .def("alloc_int",
Y
Yu Yang 已提交
75
           [](Tensor &self, paddle::platform::GPUPlace &place) {
Q
qijun 已提交
76
             self.mutable_data<int>(place);
Q
qijun 已提交
77
           })
Y
Yu Yang 已提交
78 79
      .def("set", PyCPUTensorSetFromArray<float>)
      .def("set", PyCPUTensorSetFromArray<int>)
80
      .def("set", PyCPUTensorSetFromArray<double>)
Q
qijun 已提交
81
#ifndef PADDLE_ONLY_CPU
Y
Yu Yang 已提交
82 83
      .def("set", PyCUDATensorSetFromArray<float>)
      .def("set", PyCUDATensorSetFromArray<int>)
84
      .def("set", PyCUDATensorSetFromArray<double>)
Q
qijun 已提交
85
#endif
86
      .def("shape", [](Tensor &self) { return vectorize(self.dims()); })
87 88 89 90 91
      .def("set_float_element", TensorSetElement<float>)
      .def("get_float_element", TensorGetElement<float>)
      .def("set_double_element", TensorSetElement<double>)
      .def("get_double_element", TensorGetElement<double>)
      .def("dtype", [](Tensor &self) { return ToDataType(self.type()); });
Y
Yu Yang 已提交
92

93
  py::class_<LoDTensor, Tensor>(m, "LoDTensor")
94 95
      .def_buffer(
          [](Tensor &self) -> py::buffer_info { return CastToPyBuffer(self); })
96 97 98
      .def(
          "__init__",
          [](LoDTensor &instance, const std::vector<std::vector<size_t>> &lod) {
99
#ifdef PADDLE_ONLY_CPU
100
            new (&instance) LoDTensor(lod);
101
#else
Y
Yu Yang 已提交
102
             LoD new_lod;
103 104
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
105
             new (&instance) LoDTensor(new_lod);
106
#endif
107
          })
D
dangqingqing 已提交
108
      .def("set_lod",
109
           [](LoDTensor &self, const std::vector<std::vector<size_t>> &lod) {
110
#ifdef PADDLE_ONLY_CPU
D
dangqingqing 已提交
111
             self.set_lod(lod);
112
#else
Y
Yu Yang 已提交
113
             LoD new_lod;
114 115 116 117
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             self.set_lod(new_lod);
#endif
D
dangqingqing 已提交
118
           })
119 120
      .def("lod", [](LoDTensor &self) -> std::vector<std::vector<size_t>> {
#ifdef PADDLE_ONLY_CPU
D
dangqingqing 已提交
121
        return self.lod();
122 123 124 125 126
#else
           auto lod = self.lod();
           std::vector<std::vector<size_t>> new_lod;
           new_lod.reserve(lod.size());
           std::transform(lod.begin(), lod.end(), std::back_inserter(new_lod),
Y
Yu Yang 已提交
127
               [](Vector<size_t> item) ->
128 129 130 131 132 133 134 135
                   std::vector<size_t> {
                 std::vector<size_t> v;
                 v.reserve(item.size());
                 std::copy(item.begin(), item.end(), std::back_inserter(v));
                 return v;
               });
           return new_lod;
#endif
D
dangqingqing 已提交
136 137
      });

138
  py::class_<Variable>(m, "Variable", R"DOC(Variable Class.
139 140 141

All parameter, weight, gradient are variables in Paddle.
)DOC")
142
      .def("is_int", [](const Variable &var) { return var.IsType<int>(); })
143
      .def("set_int",
144 145
           [](Variable &var, int val) -> void { *var.GetMutable<int>() = val; })
      .def("get_int", [](const Variable &var) -> int { return var.Get<int>(); })
146 147 148 149 150 151 152
      .def("is_float", [](const Variable &var) { return var.IsType<float>(); })
      .def("set_float",
           [](Variable &var, float val) -> void {
             *var.GetMutable<float>() = val;
           })
      .def("get_float",
           [](const Variable &var) -> float { return var.Get<float>(); })
Y
Yu Yang 已提交
153
      .def("get_tensor",
154 155
           [](Variable &self) -> LoDTensor * {
             return self.GetMutable<LoDTensor>();
D
dangqingqing 已提交
156 157
           },
           py::return_value_policy::reference)
Y
Yan Chunwei 已提交
158
      .def("get_net",
D
dongzhihong 已提交
159 160
           [](Variable &self) -> operators::NetOp * {
             return self.GetMutable<operators::NetOp>();
Y
Yan Chunwei 已提交
161
           },
Y
Yu Yang 已提交
162
           py::return_value_policy::reference);
163

164
  py::class_<Scope>(m, "Scope", "")
Y
Yu Yang 已提交
165
      .def("new_var",
166
           [](Scope &self, const std::string &name) -> Variable * {
Y
Yu Yang 已提交
167 168
             return self.NewVar(name);
           },
169
           py::return_value_policy::reference)
170
      .def("find_var", &Scope::FindVar, py::return_value_policy::reference)
Y
Yu Yang 已提交
171
      .def(py::init<>())
172
      .def("new_scope", [](Scope &self) -> Scope * { return &self.NewScope(); },
173
           py::return_value_policy::reference)
174
      .def("drop_kids", &Scope::DropKids);
175

Y
Yu Yang 已提交
176 177
  //! @note: Be careful! PyBind will return std::string as an unicode, not
  //! Python str. If you want a str object, you should cast them in Python.
Y
Yu Yang 已提交
178 179
  m.def("get_all_op_protos", []() -> std::vector<py::bytes> {
    std::vector<py::bytes> ret_values;
Y
Yu Yang 已提交
180 181 182 183

    OpInfoMap::Instance().IterAllInfo([&ret_values](const std::string &type,
                                                    const OpInfo &info) {
      if (!info.HasOpProtoAndChecker()) return;
Y
Yu Yang 已提交
184
      std::string str;
Y
Yu Yang 已提交
185
      PADDLE_ENFORCE(info.Proto().SerializeToString(&str),
Y
Yu Yang 已提交
186
                     "Serialize OpProto Error. This could be a bug of Paddle.");
Y
Yu Yang 已提交
187 188
      ret_values.emplace_back(str);
    });
Y
Yu Yang 已提交
189 190
    return ret_values;
  });
191 192 193
  m.def_submodule(
       "var_names",
       "The module will return special predefined variable name in Paddle")
Y
Yi Wang 已提交
194 195
      .def("empty", []() { return kEmptyVarName; })
      .def("temp", []() { return kTempVarName; });
Q
qijun 已提交
196
  // clang-format off
Y
Yu Yang 已提交
197
  py::class_<paddle::platform::DeviceContext>(m, "DeviceContext")
Q
qijun 已提交
198 199
      .def_static("create",
                  [](paddle::platform::CPUPlace& place)
Q
qijun 已提交
200
                      -> paddle::platform::DeviceContext* {
Q
qijun 已提交
201 202 203 204 205
                    return new paddle::platform::CPUDeviceContext();
                  })
      .def_static("create",
                  [](paddle::platform::GPUPlace& place)
                      -> paddle::platform::DeviceContext* {
Q
qijun 已提交
206
#ifdef PADDLE_ONLY_CPU
Q
qijun 已提交
207
                    PADDLE_THROW("GPUPlace is not supported in CPU device.");
Q
qijun 已提交
208
#else
Q
qijun 已提交
209
                    return new paddle::platform::CUDADeviceContext(place);
Q
qijun 已提交
210
#endif
Q
qijun 已提交
211
                  });
Q
qijun 已提交
212
  // clang-format on
Q
qijun 已提交
213

214 215 216
  py::class_<platform::GPUPlace>(m, "GPUPlace")
      .def(py::init<int>())
      .def("__str__", string::to_string<const platform::GPUPlace &>);
Q
qijun 已提交
217

218 219 220
  py::class_<paddle::platform::CPUPlace>(m, "CPUPlace")
      .def(py::init<>())
      .def("__str__", string::to_string<const platform::CPUPlace &>);
Y
Yu Yang 已提交
221

Y
Yu Yang 已提交
222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237
  py::class_<OperatorBase>(m, "Operator")
      .def_static("create",
                  [](py::bytes protobin) {
                    OpDesc desc;
                    PADDLE_ENFORCE(desc.ParsePartialFromString(protobin),
                                   "Cannot parse user input to OpDesc");
                    PADDLE_ENFORCE(desc.IsInitialized(),
                                   "User OpDesc is not initialized, reason %s",
                                   desc.InitializationErrorString());
                    return OpRegistry::CreateOp(desc);
                  })
      .def("backward",
           [](const OperatorBase &forwardOp,
              const std::unordered_set<std::string> &no_grad_vars) {
             return Backward(forwardOp, no_grad_vars).release();
           })
238
      .def("run",
239
           [](OperatorBase &self, const Scope &scope,
240 241 242 243
              const platform::DeviceContext &dev_ctx) {
             self.Run(scope, dev_ctx);
             dev_ctx.Wait();
           })
Y
Yu Yang 已提交
244 245 246 247 248 249 250
      .def("type",
           [](const OperatorBase &op) -> std::string { return op.Type(); })
      .def("outputs",
           [](const OperatorBase &op)
               -> std::map<std::string, std::vector<std::string>> {
                 return op.Outputs();
               })
Q
qijun 已提交
251 252
      .def("output_vars",
           [](const OperatorBase &op) { return op.OutputVars(true); })
Y
Yu Yang 已提交
253
      .def("inputs", [](const OperatorBase &op) { return op.Inputs(); })
Q
qijun 已提交
254
      .def("input_vars", [](const OperatorBase &op) { return op.InputVars(); })
Y
Yu Yang 已提交
255 256 257 258
      .def("__str__", &OperatorBase::DebugString)
      .def("no_intermediate_outputs",
           [](const OperatorBase &op) { return op.OutputVars(false); })
      .def("support_gpu", &OperatorBase::SupportGPU);
Y
Yu Yang 已提交
259

Y
Yu Yang 已提交
260 261 262 263 264 265 266
  py::class_<operators::NetOp, OperatorBase>(m, "Net")
      .def_static("create",
                  []() -> operators::NetOp * {
                    auto *retv = new operators::NetOp;
                    retv->SetType("plain_net");
                    return retv;
                  })
267 268
      .def("append_op", [](operators::NetOp &self,
                           const OperatorBase &op) { self.AppendOp(op); })
D
dongzhihong 已提交
269 270 271 272
      .def("complete_add_op", &operators::NetOp::CompleteAddOp)
      .def("complete_add_op", [](std::shared_ptr<operators::NetOp> &self) {
        self->CompleteAddOp();
      });
Y
Yan Chunwei 已提交
273

Y
Yan Chunwei 已提交
274
  // recurrent_op
Y
Yu Yang 已提交
275 276 277 278 279 280 281 282 283 284 285 286 287
  py::class_<operators::RecurrentOp, OperatorBase>(m, "RecurrentOp")
      .def_static(
          "create",
          [](py::bytes protobin) -> operators::RecurrentOp * {
            OpDesc desc;
            PADDLE_ENFORCE(desc.ParsePartialFromString(protobin),
                           "Cannot parse user input to OpDesc");
            PADDLE_ENFORCE(desc.IsInitialized(),
                           "User OpDesc is not initialized, reason %s",
                           desc.InitializationErrorString());
            auto rnn_op = OpRegistry::CreateOp(desc);
            return static_cast<operators::RecurrentOp *>(rnn_op.release());
          })
288 289 290 291
      .def("set_stepnet", [](operators::RecurrentOp &self,
                             const operators::NetOp &net) -> void {
        self.set_stepnet(net.Clone());
      });
Y
Yan Chunwei 已提交
292

Z
cond op  
zchen0211 已提交
293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314
  // cond_op
  py::class_<operators::CondOp, OperatorBase>(m, "CondOp")
      .def_static("create",
                  [](py::bytes protobin) -> operators::CondOp * {
                    OpDesc desc;
                    PADDLE_ENFORCE(desc.ParsePartialFromString(protobin),
                                   "Cannot parse user input to OpDesc");
                    PADDLE_ENFORCE(desc.IsInitialized(),
                                   "User OpDesc is not initialized, reason %s",
                                   desc.InitializationErrorString());
                    auto cond_op = OpRegistry::CreateOp(desc);
                    return static_cast<operators::CondOp *>(cond_op.release());
                  })
      .def("set_truenet",
           [](operators::CondOp &self, const operators::NetOp &net) -> void {
             self.set_truenet(net.Clone());
           })
      .def("set_falsenet",
           [](operators::CondOp &self, const operators::NetOp &net) -> void {
             self.set_falsenet(net.Clone());
           });

315 316
  m.def("unique_integer", UniqueIntegerGenerator);

Q
qijun 已提交
317 318
  m.def("is_compile_gpu", IsCompileGPU);

F
fengjiayi 已提交
319 320 321 322
  BindProgramDesc(m);
  BindBlockDesc(m);
  BindVarDsec(m);
  BindOpDesc(m);
Y
Yu Yang 已提交
323

324
  return m.ptr();
L
Luo Tao 已提交
325
}
326
}  // namespace pybind
327
}  // namespace paddle