pybind.cc 16.4 KB
Newer Older
1 2 3 4 5 6
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

7
http://www.apache.org/licenses/LICENSE-2.0
8 9 10 11 12 13 14

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

L
Luo Tao 已提交
15
#include <Python.h>
Y
Yu Yang 已提交
16
#include <fstream>
Y
Yu Yang 已提交
17
#include <vector>
18

Q
Qiao Longfei 已提交
19
#include "paddle/framework/backward.h"
D
dangqingqing 已提交
20
#include "paddle/framework/lod_tensor.h"
Y
Yu Yang 已提交
21
#include "paddle/framework/op_registry.h"
Z
zchen0211 已提交
22
#include "paddle/operators/cond_op.h"
Y
Yan Chunwei 已提交
23
#include "paddle/operators/net_op.h"
Y
Yan Chunwei 已提交
24
#include "paddle/operators/recurrent_op.h"
Q
qijun 已提交
25
#include "paddle/platform/enforce.h"
Q
qijun 已提交
26
#include "paddle/platform/place.h"
L
Luo Tao 已提交
27
#include "paddle/pybind/pybind.h"
28
#include "paddle/pybind/tensor_py.h"
29
#include "paddle/string/to_string.h"
Y
Yu Yang 已提交
30 31 32 33
#include "pybind11/numpy.h"
#include "pybind11/pybind11.h"
#include "pybind11/stl.h"

34 35
namespace py = pybind11;

36 37
namespace paddle {
namespace framework {
D
dongzhihong 已提交
38 39

using Tensor = framework::Tensor;
40 41
using LoDTensor = framework::LoDTensor;
using LoD = framework::LoD;
D
dongzhihong 已提交
42

43 44 45 46 47
static size_t UniqueIntegerGenerator() {
  static std::atomic<size_t> generator;
  return generator.fetch_add(1);
}

Q
qijun 已提交
48 49 50 51 52 53 54 55
bool IsCompileGPU() {
#ifdef PADDLE_ONLY_CPU
  return false;
#else
  return true;
#endif
}

56
PYBIND11_PLUGIN(core) {
Y
Yu Yang 已提交
57
  py::module m("core", "C++ core of PaddlePaddle");
58

59 60 61
  py::class_<Tensor>(m, "Tensor", py::buffer_protocol())
      .def_buffer(
          [](Tensor &self) -> py::buffer_info { return CastToPyBuffer(self); })
Y
Yu Yang 已提交
62
      .def("get_dims",
63
           [](const Tensor &self) { return vectorize(self.dims()); })
Y
Yu Yang 已提交
64
      .def("set_dims",
Q
qijun 已提交
65
           [](Tensor &self, const std::vector<int64_t> &dim) {
66
             self.Resize(make_ddim(dim));
Y
Yu Yang 已提交
67 68
           })
      .def("alloc_float",
Y
Yu Yang 已提交
69
           [](Tensor &self, paddle::platform::GPUPlace &place) {
Q
qijun 已提交
70
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
71
           })
Q
qijun 已提交
72
      .def("alloc_float",
Y
Yu Yang 已提交
73
           [](Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
74
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
75 76
           })
      .def("alloc_int",
Y
Yu Yang 已提交
77
           [](Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
78
             self.mutable_data<int>(place);
Y
Yu Yang 已提交
79
           })
Q
qijun 已提交
80
      .def("alloc_int",
Y
Yu Yang 已提交
81
           [](Tensor &self, paddle::platform::GPUPlace &place) {
Q
qijun 已提交
82
             self.mutable_data<int>(place);
Q
qijun 已提交
83
           })
Y
Yu Yang 已提交
84 85
      .def("set", PyCPUTensorSetFromArray<float>)
      .def("set", PyCPUTensorSetFromArray<int>)
Q
qijun 已提交
86
#ifndef PADDLE_ONLY_CPU
Y
Yu Yang 已提交
87 88
      .def("set", PyCUDATensorSetFromArray<float>)
      .def("set", PyCUDATensorSetFromArray<int>)
Q
qijun 已提交
89
#endif
90
      .def("shape", [](Tensor &self) { return vectorize(self.dims()); })
Y
Yu Yang 已提交
91
      .def("set_float_element",
92
           [](Tensor &self, size_t offset, float f) {
Y
Yu Yang 已提交
93
             // TODO(yuyang18): Only support GPU now.
Y
Yu Yang 已提交
94 95
             self.data<float>()[offset] = f;
           })
96
      .def("get_float_element", [](Tensor &self, size_t offset) -> float {
Y
Yu Yang 已提交
97
        // TODO(yuyang18): Only support GPU now.
Y
Yu Yang 已提交
98 99
        return self.data<float>()[offset];
      });
Y
Yu Yang 已提交
100

101
  py::class_<LoDTensor, Tensor>(m, "LoDTensor")
102 103
      .def_buffer(
          [](Tensor &self) -> py::buffer_info { return CastToPyBuffer(self); })
104 105 106
      .def(
          "__init__",
          [](LoDTensor &instance, const std::vector<std::vector<size_t>> &lod) {
107
#ifdef PADDLE_ONLY_CPU
108
            new (&instance) LoDTensor(lod);
109 110 111 112
#else
             paddle::framework::LoD new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
113
             new (&instance) LoDTensor(new_lod);
114
#endif
115
          })
D
dangqingqing 已提交
116
      .def("set_lod",
117
           [](LoDTensor &self, const std::vector<std::vector<size_t>> &lod) {
118
#ifdef PADDLE_ONLY_CPU
D
dangqingqing 已提交
119
             self.set_lod(lod);
120 121 122 123 124 125
#else
             paddle::framework::LoD new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             self.set_lod(new_lod);
#endif
D
dangqingqing 已提交
126
           })
127 128
      .def("lod", [](LoDTensor &self) -> std::vector<std::vector<size_t>> {
#ifdef PADDLE_ONLY_CPU
D
dangqingqing 已提交
129
        return self.lod();
130 131 132 133 134 135 136 137 138 139 140 141 142 143
#else
           auto lod = self.lod();
           std::vector<std::vector<size_t>> new_lod;
           new_lod.reserve(lod.size());
           std::transform(lod.begin(), lod.end(), std::back_inserter(new_lod),
               [](paddle::framework::Vector<size_t> item) ->
                   std::vector<size_t> {
                 std::vector<size_t> v;
                 v.reserve(item.size());
                 std::copy(item.begin(), item.end(), std::back_inserter(v));
                 return v;
               });
           return new_lod;
#endif
D
dangqingqing 已提交
144 145
      });

146
  py::class_<Variable>(m, "Variable", R"DOC(Variable Class.
147 148 149

All parameter, weight, gradient are variables in Paddle.
)DOC")
150
      .def("is_int", [](const Variable &var) { return var.IsType<int>(); })
151
      .def("set_int",
152 153
           [](Variable &var, int val) -> void { *var.GetMutable<int>() = val; })
      .def("get_int", [](const Variable &var) -> int { return var.Get<int>(); })
Y
Yu Yang 已提交
154
      .def("get_tensor",
155 156
           [](Variable &self) -> LoDTensor * {
             return self.GetMutable<LoDTensor>();
D
dangqingqing 已提交
157 158
           },
           py::return_value_policy::reference)
Y
Yan Chunwei 已提交
159
      .def("get_net",
D
dongzhihong 已提交
160 161
           [](Variable &self) -> operators::NetOp * {
             return self.GetMutable<operators::NetOp>();
Y
Yan Chunwei 已提交
162
           },
Y
Yu Yang 已提交
163
           py::return_value_policy::reference);
164

165
  py::class_<Scope>(m, "Scope", "")
Y
Yu Yang 已提交
166
      .def("new_var",
167
           [](Scope &self, const std::string &name) -> Variable * {
Y
Yu Yang 已提交
168 169
             return self.NewVar(name);
           },
170
           py::return_value_policy::reference)
171
      .def("find_var", &Scope::FindVar, py::return_value_policy::reference)
Y
Yu Yang 已提交
172
      .def(py::init<>())
173 174
      .def("new_scope",
           [](Scope &self) -> Scope * { return &self.NewScope(); },
175
           py::return_value_policy::reference)
176
      .def("drop_kids", &Scope::DropKids);
177

Y
Yu Yang 已提交
178 179
  //! @note: Be careful! PyBind will return std::string as an unicode, not
  //! Python str. If you want a str object, you should cast them in Python.
Y
Yu Yang 已提交
180 181
  m.def("get_all_op_protos", []() -> std::vector<py::bytes> {
    std::vector<py::bytes> ret_values;
Y
Yu Yang 已提交
182 183 184 185

    OpInfoMap::Instance().IterAllInfo([&ret_values](const std::string &type,
                                                    const OpInfo &info) {
      if (!info.HasOpProtoAndChecker()) return;
Y
Yu Yang 已提交
186
      std::string str;
Y
Yu Yang 已提交
187
      PADDLE_ENFORCE(info.Proto().SerializeToString(&str),
Y
Yu Yang 已提交
188
                     "Serialize OpProto Error. This could be a bug of Paddle.");
Y
Yu Yang 已提交
189 190
      ret_values.emplace_back(str);
    });
Y
Yu Yang 已提交
191 192
    return ret_values;
  });
193 194 195
  m.def_submodule(
       "var_names",
       "The module will return special predefined variable name in Paddle")
Y
Yi Wang 已提交
196 197
      .def("empty", []() { return kEmptyVarName; })
      .def("temp", []() { return kTempVarName; });
Q
qijun 已提交
198
  // clang-format off
Y
Yu Yang 已提交
199
  py::class_<paddle::platform::DeviceContext>(m, "DeviceContext")
Q
qijun 已提交
200 201
      .def_static("create",
                  [](paddle::platform::CPUPlace& place)
Q
qijun 已提交
202
                      -> paddle::platform::DeviceContext* {
Q
qijun 已提交
203 204 205 206 207
                    return new paddle::platform::CPUDeviceContext();
                  })
      .def_static("create",
                  [](paddle::platform::GPUPlace& place)
                      -> paddle::platform::DeviceContext* {
Q
qijun 已提交
208
#ifdef PADDLE_ONLY_CPU
Q
qijun 已提交
209
                    PADDLE_THROW("GPUPlace is not supported in CPU device.");
Q
qijun 已提交
210
#else
Q
qijun 已提交
211
                    return new paddle::platform::CUDADeviceContext(place);
Q
qijun 已提交
212
#endif
Q
qijun 已提交
213
                  });
Q
qijun 已提交
214
  // clang-format on
Q
qijun 已提交
215

216 217 218
  py::class_<platform::GPUPlace>(m, "GPUPlace")
      .def(py::init<int>())
      .def("__str__", string::to_string<const platform::GPUPlace &>);
Q
qijun 已提交
219

220 221 222
  py::class_<paddle::platform::CPUPlace>(m, "CPUPlace")
      .def(py::init<>())
      .def("__str__", string::to_string<const platform::CPUPlace &>);
Y
Yu Yang 已提交
223

Y
Yu Yang 已提交
224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248
  py::class_<OperatorBase>(m, "Operator")
      .def_static("create",
                  [](py::bytes protobin) {
                    OpDesc desc;
                    PADDLE_ENFORCE(desc.ParsePartialFromString(protobin),
                                   "Cannot parse user input to OpDesc");
                    PADDLE_ENFORCE(desc.IsInitialized(),
                                   "User OpDesc is not initialized, reason %s",
                                   desc.InitializationErrorString());
                    return OpRegistry::CreateOp(desc);
                  })
      .def("backward",
           [](const OperatorBase &forwardOp,
              const std::unordered_set<std::string> &no_grad_vars) {
             return Backward(forwardOp, no_grad_vars).release();
           })
      .def("infer_shape", &OperatorBase::InferShape)
      .def("run", &OperatorBase::Run)
      .def("type",
           [](const OperatorBase &op) -> std::string { return op.Type(); })
      .def("outputs",
           [](const OperatorBase &op)
               -> std::map<std::string, std::vector<std::string>> {
                 return op.Outputs();
               })
Q
qijun 已提交
249 250
      .def("output_vars",
           [](const OperatorBase &op) { return op.OutputVars(true); })
Y
Yu Yang 已提交
251
      .def("inputs", [](const OperatorBase &op) { return op.Inputs(); })
Q
qijun 已提交
252
      .def("input_vars", [](const OperatorBase &op) { return op.InputVars(); })
Y
Yu Yang 已提交
253 254 255 256
      .def("__str__", &OperatorBase::DebugString)
      .def("no_intermediate_outputs",
           [](const OperatorBase &op) { return op.OutputVars(false); })
      .def("support_gpu", &OperatorBase::SupportGPU);
Y
Yu Yang 已提交
257

Y
Yu Yang 已提交
258 259 260 261 262 263 264
  py::class_<operators::NetOp, OperatorBase>(m, "Net")
      .def_static("create",
                  []() -> operators::NetOp * {
                    auto *retv = new operators::NetOp;
                    retv->SetType("plain_net");
                    return retv;
                  })
265 266 267 268
      .def("append_op",
           [](operators::NetOp &self, const OperatorBase &op) {
             self.AppendOp(op);
           })
D
dongzhihong 已提交
269 270 271 272
      .def("complete_add_op", &operators::NetOp::CompleteAddOp)
      .def("complete_add_op", [](std::shared_ptr<operators::NetOp> &self) {
        self->CompleteAddOp();
      });
Y
Yan Chunwei 已提交
273

Y
Yan Chunwei 已提交
274
  // recurrent_op
Y
Yu Yang 已提交
275 276 277 278 279 280 281 282 283 284 285 286 287
  py::class_<operators::RecurrentOp, OperatorBase>(m, "RecurrentOp")
      .def_static(
          "create",
          [](py::bytes protobin) -> operators::RecurrentOp * {
            OpDesc desc;
            PADDLE_ENFORCE(desc.ParsePartialFromString(protobin),
                           "Cannot parse user input to OpDesc");
            PADDLE_ENFORCE(desc.IsInitialized(),
                           "User OpDesc is not initialized, reason %s",
                           desc.InitializationErrorString());
            auto rnn_op = OpRegistry::CreateOp(desc);
            return static_cast<operators::RecurrentOp *>(rnn_op.release());
          })
288 289 290
      .def("set_stepnet",
           [](operators::RecurrentOp &self, const operators::NetOp &net)
               -> void { self.set_stepnet(net.Clone()); });
Y
Yan Chunwei 已提交
291

Z
cond op  
zchen0211 已提交
292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313
  // cond_op
  py::class_<operators::CondOp, OperatorBase>(m, "CondOp")
      .def_static("create",
                  [](py::bytes protobin) -> operators::CondOp * {
                    OpDesc desc;
                    PADDLE_ENFORCE(desc.ParsePartialFromString(protobin),
                                   "Cannot parse user input to OpDesc");
                    PADDLE_ENFORCE(desc.IsInitialized(),
                                   "User OpDesc is not initialized, reason %s",
                                   desc.InitializationErrorString());
                    auto cond_op = OpRegistry::CreateOp(desc);
                    return static_cast<operators::CondOp *>(cond_op.release());
                  })
      .def("set_truenet",
           [](operators::CondOp &self, const operators::NetOp &net) -> void {
             self.set_truenet(net.Clone());
           })
      .def("set_falsenet",
           [](operators::CondOp &self, const operators::NetOp &net) -> void {
             self.set_falsenet(net.Clone());
           });

314 315
  m.def("unique_integer", UniqueIntegerGenerator);

Q
qijun 已提交
316 317
  m.def("is_compile_gpu", IsCompileGPU);

Y
Yu Yang 已提交
318
  py::class_<ProgramDesc>(m, "ProgramDesc", "")
Y
Update  
Yu Yang 已提交
319 320 321
      .def_static("instance",
                  [] { return &GetProgramDesc(); },
                  py::return_value_policy::reference)
Y
Yu Yang 已提交
322 323 324 325 326 327 328 329 330
      .def_static("__create_program_desc__",
                  [] {
                    // Only used for unit-test
                    auto *prog_desc = new ProgramDesc;
                    auto *block = prog_desc->mutable_blocks()->Add();
                    block->set_idx(0);
                    block->set_parent_idx(-1);
                    return prog_desc;
                  })
Y
Update  
Yu Yang 已提交
331 332
      .def("append_block",
           [](ProgramDesc &self, BlockDesc &parent) {
Y
Yu Yang 已提交
333
             auto desc = self.add_blocks();
Y
Update  
Yu Yang 已提交
334 335 336
             desc->set_idx(self.mutable_blocks()->size() - 1);
             desc->set_parent_idx(parent.idx());
             return desc;
Y
Yu Yang 已提交
337 338
           },
           py::return_value_policy::reference)
Y
Update  
Yu Yang 已提交
339
      .def("root_block",
Y
Yu Yang 已提交
340 341 342 343
           [](ProgramDesc &self) { return self.mutable_blocks()->Mutable(0); },
           py::return_value_policy::reference)
      .def("__str__", [](ProgramDesc &self) { return self.DebugString(); });

Y
Yu Yang 已提交
344
  py::class_<BlockDesc>(m, "BlockDesc", "")
Y
Yu Yang 已提交
345
      .def("id", [](BlockDesc &self) { return self.idx(); })
Y
Update  
Yu Yang 已提交
346
      .def("parent", [](BlockDesc &self) { return self.parent_idx(); })
F
fengjiayi 已提交
347 348
      .def("append_op", [](BlockDesc &self) { return self.add_ops(); })
      .def("new_var", [](BlockDesc &self) { return self.add_vars(); });
Y
Yu Yang 已提交
349

F
fengjiayi 已提交
350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374
  py::class_<VarDesc>(m, "VarDesc", "")
      .def(py::init<>())
      .def("set_name",
           [](VarDesc &self, const std::string &name) { self.set_name(name); })
      .def("set_shape",
           [](VarDesc &self, const std::vector<int64_t> &dims) {
             LoDTensorDesc *lod_tensor_desc = self.mutable_lod_tensor();
             for (const int64_t &i : dims) {
               lod_tensor_desc->add_dims(i);
             }
           })
      .def("set_data_type",
           [](VarDesc &self, int type_id) {
             LoDTensorDesc *lod_tensor_desc = self.mutable_lod_tensor();
             lod_tensor_desc->set_data_type(static_cast<DataType>(type_id));
           })
      .def("shape", [](VarDesc &self) {
        const LoDTensorDesc &lod_tensor_desc = self.lod_tensor();
        int rank = lod_tensor_desc.dims_size();
        std::vector<int64_t> res(rank);
        for (int i = 0; i < rank; ++i) {
          res[i] = lod_tensor_desc.dims(i);
        }
        return res;
      });
Y
Yu Yang 已提交
375

Y
Update  
Yu Yang 已提交
376 377 378 379 380 381 382 383 384 385 386 387
  auto op_desc_set_var = [](OpDesc::Var *var,
                            const std::string &parameter,
                            const std::vector<std::string> &arguments) {
    var->set_parameter(parameter);
    auto args = var->mutable_arguments();
    args->Reserve(static_cast<int>(arguments.size()));
    for (auto &arg : arguments) {
      *args->Add() = arg;
    }
  };

  auto op_desc_set_attr = [](OpDesc &desc, const std::string &name) {
F
fengjiayi 已提交
388
    auto attr = desc.add_attrs();
Y
Update  
Yu Yang 已提交
389 390 391 392 393 394 395 396 397 398
    attr->set_name(name);
    return attr;
  };

  py::class_<OpDesc>(m, "OpDesc", "")
      .def("type", [](OpDesc &op) { return op.type(); })
      .def("set_input",
           [op_desc_set_var](OpDesc &self,
                             const std::string &parameter,
                             const std::vector<std::string> &arguments) {
F
fengjiayi 已提交
399
             auto ipt = self.add_inputs();
Y
Update  
Yu Yang 已提交
400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417
             op_desc_set_var(ipt, parameter, arguments);
           })
      .def("input_names",
           [](OpDesc &self) {
             std::vector<std::string> ret_val;
             ret_val.reserve(static_cast<size_t>(self.inputs().size()));
             std::transform(
                 self.inputs().begin(),
                 self.inputs().end(),
                 std::back_inserter(ret_val),
                 [](const OpDesc::Var &var) { return var.parameter(); });
             return ret_val;
           })
      .def("__str__", [](OpDesc &self) { return self.DebugString(); })
      .def("set_output",
           [op_desc_set_var](OpDesc &self,
                             const std::string &parameter,
                             const std::vector<std::string> &arguments) {
F
fengjiayi 已提交
418
             auto opt = self.add_outputs();
Y
Update  
Yu Yang 已提交
419 420 421 422 423 424
             op_desc_set_var(opt, parameter, arguments);
           })
      .def("set_attr",
           [op_desc_set_attr](OpDesc &self, const std::string &name, int i) {
             op_desc_set_attr(self, name)->set_i(i);
           });
Y
Yu Yang 已提交
425

426
  return m.ptr();
L
Luo Tao 已提交
427
}
428 429
}  // namespace framework
}  // namespace paddle