pybind.cc 12.1 KB
Newer Older
1 2 3 4 5 6
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

7
http://www.apache.org/licenses/LICENSE-2.0
8 9 10 11 12 13 14

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

L
Luo Tao 已提交
15
#include <Python.h>
Y
Yu Yang 已提交
16
#include <fstream>
Y
Yu Yang 已提交
17
#include <vector>
18

Q
Qiao Longfei 已提交
19
#include "paddle/framework/backward.h"
D
dangqingqing 已提交
20
#include "paddle/framework/lod_tensor.h"
Y
Yu Yang 已提交
21
#include "paddle/framework/op_registry.h"
Z
zchen0211 已提交
22
#include "paddle/operators/cond_op.h"
Y
Yan Chunwei 已提交
23
#include "paddle/operators/net_op.h"
Y
Yan Chunwei 已提交
24
#include "paddle/operators/recurrent_op.h"
Q
qijun 已提交
25
#include "paddle/platform/enforce.h"
Q
qijun 已提交
26
#include "paddle/platform/place.h"
L
Luo Tao 已提交
27
#include "paddle/pybind/pybind.h"
28
#include "paddle/pybind/tensor_py.h"
29
#include "paddle/string/to_string.h"
Y
Yu Yang 已提交
30 31 32 33
#include "pybind11/numpy.h"
#include "pybind11/pybind11.h"
#include "pybind11/stl.h"

34 35
namespace py = pybind11;

36
namespace paddle {
37
namespace pybind {
38 39 40 41 42
static size_t UniqueIntegerGenerator() {
  static std::atomic<size_t> generator;
  return generator.fetch_add(1);
}

Q
qijun 已提交
43 44 45 46 47 48 49 50
bool IsCompileGPU() {
#ifdef PADDLE_ONLY_CPU
  return false;
#else
  return true;
#endif
}

51
PYBIND11_PLUGIN(core) {
Y
Yu Yang 已提交
52
  py::module m("core", "C++ core of PaddlePaddle");
53

54 55 56 57
  // using framework in this function. Since it is inside a function, it will
  // not cause namespace pollution.
  using namespace paddle::framework;  // NOLINT

58 59 60
  py::class_<Tensor>(m, "Tensor", py::buffer_protocol())
      .def_buffer(
          [](Tensor &self) -> py::buffer_info { return CastToPyBuffer(self); })
Y
Yu Yang 已提交
61
      .def("get_dims",
62
           [](const Tensor &self) { return vectorize(self.dims()); })
Y
Yu Yang 已提交
63
      .def("set_dims",
Q
qijun 已提交
64
           [](Tensor &self, const std::vector<int64_t> &dim) {
65
             self.Resize(framework::make_ddim(dim));
Y
Yu Yang 已提交
66 67
           })
      .def("alloc_float",
Y
Yu Yang 已提交
68
           [](Tensor &self, paddle::platform::GPUPlace &place) {
Q
qijun 已提交
69
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
70
           })
Q
qijun 已提交
71
      .def("alloc_float",
Y
Yu Yang 已提交
72
           [](Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
73
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
74 75
           })
      .def("alloc_int",
Y
Yu Yang 已提交
76
           [](Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
77
             self.mutable_data<int>(place);
Y
Yu Yang 已提交
78
           })
Q
qijun 已提交
79
      .def("alloc_int",
Y
Yu Yang 已提交
80
           [](Tensor &self, paddle::platform::GPUPlace &place) {
Q
qijun 已提交
81
             self.mutable_data<int>(place);
Q
qijun 已提交
82
           })
Y
Yu Yang 已提交
83 84
      .def("set", PyCPUTensorSetFromArray<float>)
      .def("set", PyCPUTensorSetFromArray<int>)
Q
qijun 已提交
85
#ifndef PADDLE_ONLY_CPU
Y
Yu Yang 已提交
86 87
      .def("set", PyCUDATensorSetFromArray<float>)
      .def("set", PyCUDATensorSetFromArray<int>)
Q
qijun 已提交
88
#endif
89
      .def("shape", [](Tensor &self) { return vectorize(self.dims()); })
Y
Yu Yang 已提交
90
      .def("set_float_element",
91
           [](Tensor &self, size_t offset, float f) {
Y
Yu Yang 已提交
92
             // TODO(yuyang18): Only support GPU now.
Y
Yu Yang 已提交
93 94
             self.data<float>()[offset] = f;
           })
95
      .def("get_float_element", [](Tensor &self, size_t offset) -> float {
Y
Yu Yang 已提交
96
        // TODO(yuyang18): Only support GPU now.
Y
Yu Yang 已提交
97 98
        return self.data<float>()[offset];
      });
Y
Yu Yang 已提交
99

100
  py::class_<LoDTensor, Tensor>(m, "LoDTensor")
101 102
      .def_buffer(
          [](Tensor &self) -> py::buffer_info { return CastToPyBuffer(self); })
103 104 105
      .def(
          "__init__",
          [](LoDTensor &instance, const std::vector<std::vector<size_t>> &lod) {
106
#ifdef PADDLE_ONLY_CPU
107
            new (&instance) LoDTensor(lod);
108 109 110 111
#else
             paddle::framework::LoD new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
112
             new (&instance) LoDTensor(new_lod);
113
#endif
114
          })
D
dangqingqing 已提交
115
      .def("set_lod",
116
           [](LoDTensor &self, const std::vector<std::vector<size_t>> &lod) {
117
#ifdef PADDLE_ONLY_CPU
D
dangqingqing 已提交
118
             self.set_lod(lod);
119 120 121 122 123 124
#else
             paddle::framework::LoD new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             self.set_lod(new_lod);
#endif
D
dangqingqing 已提交
125
           })
126 127
      .def("lod", [](LoDTensor &self) -> std::vector<std::vector<size_t>> {
#ifdef PADDLE_ONLY_CPU
D
dangqingqing 已提交
128
        return self.lod();
129 130 131 132 133 134 135 136 137 138 139 140 141 142
#else
           auto lod = self.lod();
           std::vector<std::vector<size_t>> new_lod;
           new_lod.reserve(lod.size());
           std::transform(lod.begin(), lod.end(), std::back_inserter(new_lod),
               [](paddle::framework::Vector<size_t> item) ->
                   std::vector<size_t> {
                 std::vector<size_t> v;
                 v.reserve(item.size());
                 std::copy(item.begin(), item.end(), std::back_inserter(v));
                 return v;
               });
           return new_lod;
#endif
D
dangqingqing 已提交
143 144
      });

145
  py::class_<Variable>(m, "Variable", R"DOC(Variable Class.
146 147 148

All parameter, weight, gradient are variables in Paddle.
)DOC")
149
      .def("is_int", [](const Variable &var) { return var.IsType<int>(); })
150
      .def("set_int",
151 152
           [](Variable &var, int val) -> void { *var.GetMutable<int>() = val; })
      .def("get_int", [](const Variable &var) -> int { return var.Get<int>(); })
Y
Yu Yang 已提交
153
      .def("get_tensor",
154 155
           [](Variable &self) -> LoDTensor * {
             return self.GetMutable<LoDTensor>();
D
dangqingqing 已提交
156 157
           },
           py::return_value_policy::reference)
Y
Yan Chunwei 已提交
158
      .def("get_net",
D
dongzhihong 已提交
159 160
           [](Variable &self) -> operators::NetOp * {
             return self.GetMutable<operators::NetOp>();
Y
Yan Chunwei 已提交
161
           },
Y
Yu Yang 已提交
162
           py::return_value_policy::reference);
163

164
  py::class_<Scope>(m, "Scope", "")
Y
Yu Yang 已提交
165
      .def("new_var",
166
           [](Scope &self, const std::string &name) -> Variable * {
Y
Yu Yang 已提交
167 168
             return self.NewVar(name);
           },
169
           py::return_value_policy::reference)
170
      .def("find_var", &Scope::FindVar, py::return_value_policy::reference)
Y
Yu Yang 已提交
171
      .def(py::init<>())
172 173
      .def("new_scope",
           [](Scope &self) -> Scope * { return &self.NewScope(); },
174
           py::return_value_policy::reference)
175
      .def("drop_kids", &Scope::DropKids);
176

Y
Yu Yang 已提交
177 178
  //! @note: Be careful! PyBind will return std::string as an unicode, not
  //! Python str. If you want a str object, you should cast them in Python.
Y
Yu Yang 已提交
179 180
  m.def("get_all_op_protos", []() -> std::vector<py::bytes> {
    std::vector<py::bytes> ret_values;
Y
Yu Yang 已提交
181 182 183 184

    OpInfoMap::Instance().IterAllInfo([&ret_values](const std::string &type,
                                                    const OpInfo &info) {
      if (!info.HasOpProtoAndChecker()) return;
Y
Yu Yang 已提交
185
      std::string str;
Y
Yu Yang 已提交
186
      PADDLE_ENFORCE(info.Proto().SerializeToString(&str),
Y
Yu Yang 已提交
187
                     "Serialize OpProto Error. This could be a bug of Paddle.");
Y
Yu Yang 已提交
188 189
      ret_values.emplace_back(str);
    });
Y
Yu Yang 已提交
190 191
    return ret_values;
  });
192 193 194
  m.def_submodule(
       "var_names",
       "The module will return special predefined variable name in Paddle")
Y
Yi Wang 已提交
195 196
      .def("empty", []() { return kEmptyVarName; })
      .def("temp", []() { return kTempVarName; });
Q
qijun 已提交
197
  // clang-format off
Y
Yu Yang 已提交
198
  py::class_<paddle::platform::DeviceContext>(m, "DeviceContext")
Q
qijun 已提交
199 200
      .def_static("create",
                  [](paddle::platform::CPUPlace& place)
Q
qijun 已提交
201
                      -> paddle::platform::DeviceContext* {
Q
qijun 已提交
202 203 204 205 206
                    return new paddle::platform::CPUDeviceContext();
                  })
      .def_static("create",
                  [](paddle::platform::GPUPlace& place)
                      -> paddle::platform::DeviceContext* {
Q
qijun 已提交
207
#ifdef PADDLE_ONLY_CPU
Q
qijun 已提交
208
                    PADDLE_THROW("GPUPlace is not supported in CPU device.");
Q
qijun 已提交
209
#else
Q
qijun 已提交
210
                    return new paddle::platform::CUDADeviceContext(place);
Q
qijun 已提交
211
#endif
Q
qijun 已提交
212
                  });
Q
qijun 已提交
213
  // clang-format on
Q
qijun 已提交
214

215 216 217
  py::class_<platform::GPUPlace>(m, "GPUPlace")
      .def(py::init<int>())
      .def("__str__", string::to_string<const platform::GPUPlace &>);
Q
qijun 已提交
218

219 220 221
  py::class_<paddle::platform::CPUPlace>(m, "CPUPlace")
      .def(py::init<>())
      .def("__str__", string::to_string<const platform::CPUPlace &>);
Y
Yu Yang 已提交
222

Y
Yu Yang 已提交
223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247
  py::class_<OperatorBase>(m, "Operator")
      .def_static("create",
                  [](py::bytes protobin) {
                    OpDesc desc;
                    PADDLE_ENFORCE(desc.ParsePartialFromString(protobin),
                                   "Cannot parse user input to OpDesc");
                    PADDLE_ENFORCE(desc.IsInitialized(),
                                   "User OpDesc is not initialized, reason %s",
                                   desc.InitializationErrorString());
                    return OpRegistry::CreateOp(desc);
                  })
      .def("backward",
           [](const OperatorBase &forwardOp,
              const std::unordered_set<std::string> &no_grad_vars) {
             return Backward(forwardOp, no_grad_vars).release();
           })
      .def("infer_shape", &OperatorBase::InferShape)
      .def("run", &OperatorBase::Run)
      .def("type",
           [](const OperatorBase &op) -> std::string { return op.Type(); })
      .def("outputs",
           [](const OperatorBase &op)
               -> std::map<std::string, std::vector<std::string>> {
                 return op.Outputs();
               })
Q
qijun 已提交
248 249
      .def("output_vars",
           [](const OperatorBase &op) { return op.OutputVars(true); })
Y
Yu Yang 已提交
250
      .def("inputs", [](const OperatorBase &op) { return op.Inputs(); })
Q
qijun 已提交
251
      .def("input_vars", [](const OperatorBase &op) { return op.InputVars(); })
Y
Yu Yang 已提交
252 253 254 255
      .def("__str__", &OperatorBase::DebugString)
      .def("no_intermediate_outputs",
           [](const OperatorBase &op) { return op.OutputVars(false); })
      .def("support_gpu", &OperatorBase::SupportGPU);
Y
Yu Yang 已提交
256

Y
Yu Yang 已提交
257 258 259 260 261 262 263
  py::class_<operators::NetOp, OperatorBase>(m, "Net")
      .def_static("create",
                  []() -> operators::NetOp * {
                    auto *retv = new operators::NetOp;
                    retv->SetType("plain_net");
                    return retv;
                  })
264 265 266 267
      .def("append_op",
           [](operators::NetOp &self, const OperatorBase &op) {
             self.AppendOp(op);
           })
D
dongzhihong 已提交
268 269 270 271
      .def("complete_add_op", &operators::NetOp::CompleteAddOp)
      .def("complete_add_op", [](std::shared_ptr<operators::NetOp> &self) {
        self->CompleteAddOp();
      });
Y
Yan Chunwei 已提交
272

Y
Yan Chunwei 已提交
273
  // recurrent_op
Y
Yu Yang 已提交
274 275 276 277 278 279 280 281 282 283 284 285 286
  py::class_<operators::RecurrentOp, OperatorBase>(m, "RecurrentOp")
      .def_static(
          "create",
          [](py::bytes protobin) -> operators::RecurrentOp * {
            OpDesc desc;
            PADDLE_ENFORCE(desc.ParsePartialFromString(protobin),
                           "Cannot parse user input to OpDesc");
            PADDLE_ENFORCE(desc.IsInitialized(),
                           "User OpDesc is not initialized, reason %s",
                           desc.InitializationErrorString());
            auto rnn_op = OpRegistry::CreateOp(desc);
            return static_cast<operators::RecurrentOp *>(rnn_op.release());
          })
287 288 289
      .def("set_stepnet",
           [](operators::RecurrentOp &self, const operators::NetOp &net)
               -> void { self.set_stepnet(net.Clone()); });
Y
Yan Chunwei 已提交
290

Z
cond op  
zchen0211 已提交
291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312
  // cond_op
  py::class_<operators::CondOp, OperatorBase>(m, "CondOp")
      .def_static("create",
                  [](py::bytes protobin) -> operators::CondOp * {
                    OpDesc desc;
                    PADDLE_ENFORCE(desc.ParsePartialFromString(protobin),
                                   "Cannot parse user input to OpDesc");
                    PADDLE_ENFORCE(desc.IsInitialized(),
                                   "User OpDesc is not initialized, reason %s",
                                   desc.InitializationErrorString());
                    auto cond_op = OpRegistry::CreateOp(desc);
                    return static_cast<operators::CondOp *>(cond_op.release());
                  })
      .def("set_truenet",
           [](operators::CondOp &self, const operators::NetOp &net) -> void {
             self.set_truenet(net.Clone());
           })
      .def("set_falsenet",
           [](operators::CondOp &self, const operators::NetOp &net) -> void {
             self.set_falsenet(net.Clone());
           });

313 314
  m.def("unique_integer", UniqueIntegerGenerator);

Q
qijun 已提交
315 316
  m.def("is_compile_gpu", IsCompileGPU);

317
  return m.ptr();
L
Luo Tao 已提交
318
}
319
}  // namespace pybind
320
}  // namespace paddle