pybind.cc 11.5 KB
Newer Older
1 2 3 4 5 6
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

7
http://www.apache.org/licenses/LICENSE-2.0
8 9 10 11 12 13 14

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

L
Luo Tao 已提交
15
#include <Python.h>
Y
Yu Yang 已提交
16
#include <fstream>
Y
Yu Yang 已提交
17
#include <vector>
18

Q
Qiao Longfei 已提交
19
#include "paddle/framework/backward.h"
D
dangqingqing 已提交
20
#include "paddle/framework/lod_tensor.h"
Y
Yu Yang 已提交
21
#include "paddle/framework/op_registry.h"
Y
Yan Chunwei 已提交
22
#include "paddle/operators/net_op.h"
Y
Yan Chunwei 已提交
23
#include "paddle/operators/recurrent_op.h"
Q
qijun 已提交
24
#include "paddle/platform/enforce.h"
Q
qijun 已提交
25
#include "paddle/platform/place.h"
L
Luo Tao 已提交
26
#include "paddle/pybind/pybind.h"
27
#include "paddle/pybind/tensor_py.h"
28
#include "paddle/string/to_string.h"
Y
Yu Yang 已提交
29 30 31 32
#include "pybind11/numpy.h"
#include "pybind11/pybind11.h"
#include "pybind11/stl.h"

33 34
namespace py = pybind11;

35 36
namespace paddle {
namespace framework {
D
dongzhihong 已提交
37 38

using Tensor = framework::Tensor;
39 40
using LoDTensor = framework::LoDTensor;
using LoD = framework::LoD;
D
dongzhihong 已提交
41

42 43 44 45 46
static size_t UniqueIntegerGenerator() {
  static std::atomic<size_t> generator;
  return generator.fetch_add(1);
}

Q
qijun 已提交
47 48 49 50 51 52 53 54
bool IsCompileGPU() {
#ifdef PADDLE_ONLY_CPU
  return false;
#else
  return true;
#endif
}

55
PYBIND11_PLUGIN(core) {
Y
Yu Yang 已提交
56
  py::module m("core", "C++ core of PaddlePaddle");
57

58 59 60
  py::class_<Tensor>(m, "Tensor", py::buffer_protocol())
      .def_buffer(
          [](Tensor &self) -> py::buffer_info { return CastToPyBuffer(self); })
Y
Yu Yang 已提交
61
      .def("get_dims",
62
           [](const Tensor &self) { return vectorize(self.dims()); })
Y
Yu Yang 已提交
63
      .def("set_dims",
Q
qijun 已提交
64
           [](Tensor &self, const std::vector<int64_t> &dim) {
65
             self.Resize(make_ddim(dim));
Y
Yu Yang 已提交
66 67
           })
      .def("alloc_float",
Y
Yu Yang 已提交
68
           [](Tensor &self, paddle::platform::GPUPlace &place) {
Q
qijun 已提交
69
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
70
           })
Q
qijun 已提交
71
      .def("alloc_float",
Y
Yu Yang 已提交
72
           [](Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
73
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
74 75
           })
      .def("alloc_int",
Y
Yu Yang 已提交
76
           [](Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
77
             self.mutable_data<int>(place);
Y
Yu Yang 已提交
78
           })
Q
qijun 已提交
79
      .def("alloc_int",
Y
Yu Yang 已提交
80
           [](Tensor &self, paddle::platform::GPUPlace &place) {
Q
qijun 已提交
81
             self.mutable_data<int>(place);
Q
qijun 已提交
82
           })
Y
Yu Yang 已提交
83 84
      .def("set", PyCPUTensorSetFromArray<float>)
      .def("set", PyCPUTensorSetFromArray<int>)
Q
qijun 已提交
85
#ifndef PADDLE_ONLY_CPU
Y
Yu Yang 已提交
86 87
      .def("set", PyCUDATensorSetFromArray<float>)
      .def("set", PyCUDATensorSetFromArray<int>)
Q
qijun 已提交
88
#endif
89
      .def("shape", [](Tensor &self) { return vectorize(self.dims()); })
Y
Yu Yang 已提交
90
      .def("set_float_element",
91
           [](Tensor &self, size_t offset, float f) {
Y
Yu Yang 已提交
92
             // TODO(yuyang18): Only support GPU now.
Y
Yu Yang 已提交
93 94
             self.data<float>()[offset] = f;
           })
95
      .def("get_float_element", [](Tensor &self, size_t offset) -> float {
Y
Yu Yang 已提交
96
        // TODO(yuyang18): Only support GPU now.
Y
Yu Yang 已提交
97 98
        return self.data<float>()[offset];
      });
Y
Yu Yang 已提交
99

100
  py::class_<LoDTensor>(m, "LoDTensor", R"DOC(LoD(Leval of Ddetails) Tensor.
D
dangqingqing 已提交
101

102
The tensor and LoD info should be created before creating the LoDTensor, then
D
dangqingqing 已提交
103 104 105
call the set_tensor and set_lod functions to set them.

)DOC")
106 107 108 109 110 111 112 113 114 115 116 117 118
      .def("__init__",
           [](LoDTensor &instance,
              const std::vector<std::vector<size_t>> &lod,
              Tensor *t) {
#ifdef PADDLE_ONLY_CPU
             new (&instance) LoDTensor(lod, t);
#else
             paddle::framework::LoD new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             new (&instance) LoDTensor(new_lod, t);
#endif
           })
D
dangqingqing 已提交
119
      .def("set_tensor",
120
           [](LoDTensor &self, Tensor *tensor) { self.set_tensor(tensor); })
D
dangqingqing 已提交
121
      .def("set_lod",
122
           [](LoDTensor &self, const std::vector<std::vector<size_t>> &lod) {
123
#ifdef PADDLE_ONLY_CPU
D
dangqingqing 已提交
124
             self.set_lod(lod);
125 126 127 128 129 130
#else
             paddle::framework::LoD new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             self.set_lod(new_lod);
#endif
D
dangqingqing 已提交
131
           })
132 133
      .def("tensor",
           [](LoDTensor &self) -> Tensor & { return self.tensor(); },
D
dangqingqing 已提交
134
           py::return_value_policy::reference)
135 136
      .def("lod", [](LoDTensor &self) -> std::vector<std::vector<size_t>> {
#ifdef PADDLE_ONLY_CPU
D
dangqingqing 已提交
137
        return self.lod();
138 139 140 141 142 143 144 145 146 147 148 149 150 151
#else
           auto lod = self.lod();
           std::vector<std::vector<size_t>> new_lod;
           new_lod.reserve(lod.size());
           std::transform(lod.begin(), lod.end(), std::back_inserter(new_lod),
               [](paddle::framework::Vector<size_t> item) ->
                   std::vector<size_t> {
                 std::vector<size_t> v;
                 v.reserve(item.size());
                 std::copy(item.begin(), item.end(), std::back_inserter(v));
                 return v;
               });
           return new_lod;
#endif
D
dangqingqing 已提交
152 153
      });

154
  py::class_<Variable>(m, "Variable", R"DOC(Variable Class.
155 156 157

All parameter, weight, gradient are variables in Paddle.
)DOC")
158
      .def("is_int", [](const Variable &var) { return var.IsType<int>(); })
159
      .def("set_int",
160 161
           [](Variable &var, int val) -> void { *var.GetMutable<int>() = val; })
      .def("get_int", [](const Variable &var) -> int { return var.Get<int>(); })
Y
Yu Yang 已提交
162
      .def("get_tensor",
163
           [](Variable &self) -> Tensor * { return self.GetMutable<Tensor>(); },
Y
Yan Chunwei 已提交
164
           py::return_value_policy::reference)
D
dangqingqing 已提交
165
      .def("get_lod_tensor",
166 167
           [](Variable &self) -> LoDTensor * {
             return self.GetMutable<LoDTensor>();
D
dangqingqing 已提交
168 169
           },
           py::return_value_policy::reference)
Y
Yan Chunwei 已提交
170
      .def("get_net",
D
dongzhihong 已提交
171 172
           [](Variable &self) -> operators::NetOp * {
             return self.GetMutable<operators::NetOp>();
Y
Yan Chunwei 已提交
173
           },
Y
Yu Yang 已提交
174
           py::return_value_policy::reference);
175

176
  py::class_<Scope>(m, "Scope", "")
Y
Yu Yang 已提交
177
      .def("new_var",
178
           [](Scope &self, const std::string &name) -> Variable * {
Y
Yu Yang 已提交
179 180
             return self.NewVar(name);
           },
181
           py::return_value_policy::reference)
182
      .def("find_var", &Scope::FindVar, py::return_value_policy::reference)
Y
Yu Yang 已提交
183
      .def(py::init<>())
184 185
      .def("new_scope",
           [](Scope &self) -> Scope * { return &self.NewScope(); },
186
           py::return_value_policy::reference)
187
      .def("drop_kids", &Scope::DropKids);
188

Y
Yu Yang 已提交
189 190
  //! @note: Be careful! PyBind will return std::string as an unicode, not
  //! Python str. If you want a str object, you should cast them in Python.
Y
Yu Yang 已提交
191 192
  m.def("get_all_op_protos", []() -> std::vector<py::bytes> {
    std::vector<py::bytes> ret_values;
Y
Yu Yang 已提交
193 194 195 196

    OpInfoMap::Instance().IterAllInfo([&ret_values](const std::string &type,
                                                    const OpInfo &info) {
      if (!info.HasOpProtoAndChecker()) return;
Y
Yu Yang 已提交
197
      std::string str;
Y
Yu Yang 已提交
198
      PADDLE_ENFORCE(info.Proto().SerializeToString(&str),
Y
Yu Yang 已提交
199
                     "Serialize OpProto Error. This could be a bug of Paddle.");
Y
Yu Yang 已提交
200 201
      ret_values.emplace_back(str);
    });
Y
Yu Yang 已提交
202 203
    return ret_values;
  });
204 205 206
  m.def_submodule(
       "var_names",
       "The module will return special predefined variable name in Paddle")
Y
Yi Wang 已提交
207 208
      .def("empty", []() { return kEmptyVarName; })
      .def("temp", []() { return kTempVarName; });
Q
qijun 已提交
209
  // clang-format off
Y
Yu Yang 已提交
210
  py::class_<paddle::platform::DeviceContext>(m, "DeviceContext")
Q
qijun 已提交
211 212
      .def_static("create",
                  [](paddle::platform::CPUPlace& place)
Q
qijun 已提交
213
                      -> paddle::platform::DeviceContext* {
Q
qijun 已提交
214 215 216 217 218
                    return new paddle::platform::CPUDeviceContext();
                  })
      .def_static("create",
                  [](paddle::platform::GPUPlace& place)
                      -> paddle::platform::DeviceContext* {
Q
qijun 已提交
219
#ifdef PADDLE_ONLY_CPU
Q
qijun 已提交
220
                    PADDLE_THROW("GPUPlace is not supported in CPU device.");
Q
qijun 已提交
221
#else
Q
qijun 已提交
222
                    return new paddle::platform::CUDADeviceContext(place);
Q
qijun 已提交
223
#endif
Q
qijun 已提交
224
                  });
Q
qijun 已提交
225
  // clang-format on
Q
qijun 已提交
226

227 228 229
  py::class_<platform::GPUPlace>(m, "GPUPlace")
      .def(py::init<int>())
      .def("__str__", string::to_string<const platform::GPUPlace &>);
Q
qijun 已提交
230

231 232 233
  py::class_<paddle::platform::CPUPlace>(m, "CPUPlace")
      .def(py::init<>())
      .def("__str__", string::to_string<const platform::CPUPlace &>);
Y
Yu Yang 已提交
234

Y
Yu Yang 已提交
235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259
  py::class_<OperatorBase>(m, "Operator")
      .def_static("create",
                  [](py::bytes protobin) {
                    OpDesc desc;
                    PADDLE_ENFORCE(desc.ParsePartialFromString(protobin),
                                   "Cannot parse user input to OpDesc");
                    PADDLE_ENFORCE(desc.IsInitialized(),
                                   "User OpDesc is not initialized, reason %s",
                                   desc.InitializationErrorString());
                    return OpRegistry::CreateOp(desc);
                  })
      .def("backward",
           [](const OperatorBase &forwardOp,
              const std::unordered_set<std::string> &no_grad_vars) {
             return Backward(forwardOp, no_grad_vars).release();
           })
      .def("infer_shape", &OperatorBase::InferShape)
      .def("run", &OperatorBase::Run)
      .def("type",
           [](const OperatorBase &op) -> std::string { return op.Type(); })
      .def("outputs",
           [](const OperatorBase &op)
               -> std::map<std::string, std::vector<std::string>> {
                 return op.Outputs();
               })
Q
qijun 已提交
260 261
      .def("output_vars",
           [](const OperatorBase &op) { return op.OutputVars(true); })
Y
Yu Yang 已提交
262
      .def("inputs", [](const OperatorBase &op) { return op.Inputs(); })
Q
qijun 已提交
263
      .def("input_vars", [](const OperatorBase &op) { return op.InputVars(); })
Y
Yu Yang 已提交
264 265 266 267
      .def("__str__", &OperatorBase::DebugString)
      .def("no_intermediate_outputs",
           [](const OperatorBase &op) { return op.OutputVars(false); })
      .def("support_gpu", &OperatorBase::SupportGPU);
Y
Yu Yang 已提交
268

Y
Yu Yang 已提交
269 270 271 272 273 274 275
  py::class_<operators::NetOp, OperatorBase>(m, "Net")
      .def_static("create",
                  []() -> operators::NetOp * {
                    auto *retv = new operators::NetOp;
                    retv->SetType("plain_net");
                    return retv;
                  })
276 277 278 279
      .def("append_op",
           [](operators::NetOp &self, const OperatorBase &op) {
             self.AppendOp(op);
           })
D
dongzhihong 已提交
280 281 282 283
      .def("complete_add_op", &operators::NetOp::CompleteAddOp)
      .def("complete_add_op", [](std::shared_ptr<operators::NetOp> &self) {
        self->CompleteAddOp();
      });
Y
Yan Chunwei 已提交
284

Y
Yan Chunwei 已提交
285
  // recurrent_op
Y
Yu Yang 已提交
286 287 288 289 290 291 292 293 294 295 296 297 298
  py::class_<operators::RecurrentOp, OperatorBase>(m, "RecurrentOp")
      .def_static(
          "create",
          [](py::bytes protobin) -> operators::RecurrentOp * {
            OpDesc desc;
            PADDLE_ENFORCE(desc.ParsePartialFromString(protobin),
                           "Cannot parse user input to OpDesc");
            PADDLE_ENFORCE(desc.IsInitialized(),
                           "User OpDesc is not initialized, reason %s",
                           desc.InitializationErrorString());
            auto rnn_op = OpRegistry::CreateOp(desc);
            return static_cast<operators::RecurrentOp *>(rnn_op.release());
          })
299 300 301
      .def("set_stepnet",
           [](operators::RecurrentOp &self, const operators::NetOp &net)
               -> void { self.set_stepnet(net.Clone()); });
Y
Yan Chunwei 已提交
302

303 304
  m.def("unique_integer", UniqueIntegerGenerator);

Q
qijun 已提交
305 306
  m.def("is_compile_gpu", IsCompileGPU);

307
  return m.ptr();
L
Luo Tao 已提交
308
}
309 310
}  // namespace framework
}  // namespace paddle