imperative.cc 33.8 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

#include "paddle/fluid/pybind/imperative.h"
16

17
#include <Python.h>
18 19 20 21
#include <pybind11/chrono.h>
#include <pybind11/complex.h>
#include <pybind11/functional.h>
#include <pybind11/stl.h>
22
#include <memory>
J
Jiabin Yang 已提交
23
#include <string>
24 25
#include <unordered_map>
#include <utility>
J
Jiabin Yang 已提交
26 27
#include <vector>
#include "paddle/fluid/imperative/backward_strategy.h"
28
#include "paddle/fluid/imperative/basic_engine.h"
29
#include "paddle/fluid/imperative/data_loader.h"
30
#include "paddle/fluid/imperative/layer.h"
J
Jiabin Yang 已提交
31
#include "paddle/fluid/imperative/nccl_context.h"
32
#include "paddle/fluid/imperative/partial_grad_engine.h"
33
#include "paddle/fluid/imperative/profiler.h"
34
#include "paddle/fluid/imperative/tracer.h"
M
minqiyang 已提交
35
#include "paddle/fluid/imperative/type_defs.h"
36
#include "paddle/fluid/memory/allocation/mmap_allocator.h"
37
#include "paddle/fluid/pybind/op_function.h"
38
#include "paddle/fluid/pybind/pybind_boost_headers.h"
L
Leo Chen 已提交
39
#include "paddle/fluid/pybind/tensor_py.h"
40

41 42 43
namespace paddle {
namespace pybind {

44 45
namespace py = ::pybind11;

46 47 48 49
class Layer : public imperative::Layer {
 public:
  using imperative::Layer::Layer;  // Inherit constructors

50 51 52 53
  std::vector<std::shared_ptr<imperative::VarBase>> Forward(
      const std::vector<std::shared_ptr<imperative::VarBase>> &inputs)
      override {
    PYBIND11_OVERLOAD(std::vector<std::shared_ptr<imperative::VarBase>>, Layer,
J
Jiabin Yang 已提交
54
                      Forward, inputs);  // NOLINT
55 56 57
  }
};

L
Leo Chen 已提交
58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80
static const platform::Place PyObjectToPlace(const py::object &place_obj) {
  if (py::isinstance<platform::CPUPlace>(place_obj)) {
    return place_obj.cast<platform::CPUPlace>();
  } else if (py::isinstance<platform::CUDAPlace>(place_obj)) {
    return place_obj.cast<platform::CUDAPlace>();
  } else if (py::isinstance<platform::CUDAPinnedPlace>(place_obj)) {
    return place_obj.cast<platform::CUDAPinnedPlace>();
  } else {
    PADDLE_THROW(platform::errors::InvalidArgument(
        "Place should be one of CPUPlace/CUDAPlace/CUDAPinnedPlace"));
  }
}

static void InitTensorForVarBase(imperative::VarBase *self,
                                 const py::array &array,
                                 const platform::Place place,
                                 bool persistable = false,
                                 bool zero_copy = false,
                                 std::string name = "") {
  if (name == "") {
    name = imperative::GetCurrentTracer()->GenerateUniqueName("generated_var");
  }
  new (self) imperative::VarBase(name);
81
  auto *tensor = self->MutableVar()->GetMutable<framework::LoDTensor>();
L
Leo Chen 已提交
82 83 84 85 86 87 88 89 90
  if (platform::is_cpu_place(place)) {
    SetTensorFromPyArray<platform::CPUPlace>(
        tensor, array, boost::get<platform::CPUPlace>(place), zero_copy);
  } else if (platform::is_gpu_place(place)) {
    SetTensorFromPyArray<platform::CUDAPlace>(
        tensor, array, boost::get<platform::CUDAPlace>(place), zero_copy);
  } else if (platform::is_cuda_pinned_place(place)) {
    SetTensorFromPyArray<platform::CUDAPinnedPlace>(
        tensor, array, boost::get<platform::CUDAPinnedPlace>(place), zero_copy);
91
  } else {
L
Leo Chen 已提交
92 93
    PADDLE_THROW(platform::errors::InvalidArgument(
        "Place should be one of CPUPlace/CUDAPlace/CUDAPinnedPlace"));
J
Jiabin Yang 已提交
94
  }
L
Leo Chen 已提交
95
  self->SetPersistable(persistable);
96 97 98 99 100 101 102 103 104
  self->SetType(framework::proto::VarType::LOD_TENSOR);
  self->SetDataType(tensor->type());
}

static void InitVarBaseFromNumpyWithKwargs(imperative::VarBase *self,
                                           const py::kwargs &kwargs) {
  PADDLE_ENFORCE_EQ(
      kwargs.contains("value"), true,
      platform::errors::InvalidArgument("Missing argument: value"));
L
Leo Chen 已提交
105 106 107 108 109 110 111 112 113 114 115 116 117

  auto persistable = kwargs.contains("persistable")
                         ? kwargs["persistable"].cast<bool>()
                         : false;
  auto array = kwargs.contains("value") ? kwargs["value"].cast<py::array>()
                                        : py::array();
  auto zero_copy =
      kwargs.contains("zero_copy") ? kwargs["zero_copy"].cast<bool>() : false;
  auto name = kwargs.contains("name") ? kwargs["name"].cast<std::string>() : "";
  auto default_place = imperative::GetCurrentTracer()->ExpectedPlace();
  auto place = kwargs.contains("place") ? PyObjectToPlace(kwargs["place"])
                                        : default_place;
  InitTensorForVarBase(self, array, place, persistable, zero_copy, name);
118
}
119

120 121 122
template <typename P>
static void InitVarBaseFromNumpyWithArg(imperative::VarBase *self,
                                        const py::array &array, const P &place,
L
Leo Chen 已提交
123 124 125 126 127 128 129 130
                                        bool persistable = false,
                                        bool zero_copy = false,
                                        std::string name = "") {
  // 0: self, 1: value, 2: place, 3: persistable, 4: zero_copy, 5: name
  if (name == "") {
    name = imperative::GetCurrentTracer()->GenerateUniqueName("generated_var");
  }
  new (self) imperative::VarBase(name);
131 132 133 134 135 136 137 138
  self->SetPersistable(persistable);
  auto *tensor = self->MutableVar()->GetMutable<framework::LoDTensor>();
  SetTensorFromPyArray<P>(tensor, array, place, zero_copy);
  self->SetType(framework::proto::VarType::LOD_TENSOR);
  self->SetDataType(tensor->type());
}

static void InitVarBaseFromNumpyWithArgDefault(imperative::VarBase *self,
L
Leo Chen 已提交
139 140 141
                                               const py::array &array) {
  auto place = imperative::GetCurrentTracer()->ExpectedPlace();
  InitTensorForVarBase(self, array, place);
142
}
143

144 145 146 147 148
static std::string GetTypeName(const imperative::VarBase &var) {
  if (var.Type() == framework::proto::VarType::RAW) {
    return "RAW";
  } else if (!var.Var().IsInitialized()) {
    return "nullptr";
149
  } else {
150
    return framework::ToTypeName(var.Var().Type());
151 152
  }
}
L
Leo Chen 已提交
153

154
using PyNameVarBaseMap = std::unordered_map<std::string, py::handle>;
155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176

template <typename T>
static T PyObjectCast(PyObject *obj) {
  try {
    return py::cast<T>(py::handle(obj));
  } catch (py::cast_error &) {
    PADDLE_THROW("Python object is not type of %s", typeid(T).name());
  }
}

// NOTE(zjl): py::handle is a very light wrapper of PyObject *.
// Unlike py::object, py::handle does not change reference count of PyObject *.
static std::vector<std::shared_ptr<imperative::VarBase>>
GetVarBaseListFromPyHandle(const py::handle &handle) {
  PyObject *py_obj = handle.ptr();  // get underlying PyObject
  // Python None is not nullptr in C++!
  if (!py_obj || py_obj == Py_None) {
    return {};
  }

  std::vector<std::shared_ptr<imperative::VarBase>> result;

177
  if (PyList_Check(py_obj)) {  // List of VarBase
178 179 180
    size_t len = PyList_GET_SIZE(py_obj);
    result.reserve(len);
    for (size_t i = 0; i < len; ++i) {
181 182 183
      PyObject *py_ivar = PyList_GET_ITEM(py_obj, i);
      PADDLE_ENFORCE_NOT_NULL(
          py_ivar, platform::errors::InvalidArgument("Python Object is NULL"));
184 185 186
      result.emplace_back(
          PyObjectCast<std::shared_ptr<imperative::VarBase>>(py_ivar));
    }
187
  } else if (PyTuple_Check(py_obj)) {  // Tuple of VarBase
188 189 190
    size_t len = PyTuple_GET_SIZE(py_obj);
    result.reserve(len);
    for (size_t i = 0; i < len; ++i) {
191 192 193
      PyObject *py_ivar = PyTuple_GET_ITEM(py_obj, i);
      PADDLE_ENFORCE_NOT_NULL(
          py_ivar, platform::errors::InvalidArgument("Python Object is NULL"));
194 195 196
      result.emplace_back(
          PyObjectCast<std::shared_ptr<imperative::VarBase>>(py_ivar));
    }
197 198 199
  } else {  // VarBase
    result.emplace_back(
        PyObjectCast<std::shared_ptr<imperative::VarBase>>(py_obj));
200 201 202 203 204
  }

  return result;
}

J
Jiabin Yang 已提交
205 206 207
static imperative::NameVarBaseMap ConvertToNameVarBaseMap(
    const PyNameVarBaseMap &map) {
  imperative::NameVarBaseMap result;
208 209 210 211 212 213
  for (auto &pair : map) {
    auto var_vec = GetVarBaseListFromPyHandle(pair.second);
    if (!var_vec.empty()) {
      result.emplace(pair.first, std::move(var_vec));
    }
  }
J
Jiabin Yang 已提交
214 215 216

  PADDLE_ENFORCE_EQ(PyErr_Occurred() == nullptr, true,
                    py::str(py::handle(PyErr_Occurred())));
217 218 219
  return result;
}

220
// Bind Methods
J
Jiabin Yang 已提交
221
void BindImperative(py::module *m_ptr) {
222 223
  auto &m = *m_ptr;

224 225
  BindOpFunctions(&m);

226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304
#ifndef _WIN32
  // Dygraph DataLoader signal handler
  m.def("_set_process_pid", [](int64_t key, pid_t pid) {
    imperative::SetLoadProcessPID(key, pid);
  });
  m.def("_erase_process_pid",
        [](int64_t key) { imperative::EraseLoadProcessPID(key); });
  m.def("_set_process_signal_handler",
        []() { imperative::SetLoadProcessSignalHandler(); });
  m.def("_throw_error_if_process_failed",
        []() { imperative::ThrowErrorIfLoadProcessFailed(); });

  // Dygraph DataLoader reader process & thread related functions
  m.def(
      "_convert_to_tensor_list",
      [](py::object &obj) -> py::list {
        // 0. input data check
        PADDLE_ENFORCE(
            py::isinstance<py::tuple>(obj) || py::isinstance<py::list>(obj),
            platform::errors::InvalidArgument(
                "The batch data read into DataLoader is illegal."
                "Expected data type is tuple or list, but received %s",
                obj.get_type()));
        py::list batch = py::cast<py::list>(obj);
        py::list tensors;
        for (size_t i = 0; i < batch.size(); ++i) {
          // 1. cast to python array
          auto array = batch[i].cast<py::array>();
          PADDLE_ENFORCE_NE(
              string::Sprintf("%s", array.dtype()).compare("object"), 0,
              platform::errors::InvalidArgument(
                  "Faild to convert input data to a regular ndarray.\n  * "
                  "Usually this means the input data contains nested "
                  "lists with different lengths.\n  * Check the reader "
                  "function passed to 'set_(sample/sample_list/batch)"
                  "_generator' to locate the data causes this issue."));
          // 2. construcct LoDTensor
          framework::LoDTensor t;
          SetTensorFromPyArray<platform::CPUPlace>(&t, array,
                                                   platform::CPUPlace(), true);
          // 3. allocate shared memory
          void *data_ptr = t.data<void>();
          size_t data_size = t.numel() * framework::SizeOfType(t.type());
          auto shared_writer_holder =
              memory::allocation::AllocateMemoryMapWriterAllocation(data_size);
          // 4. maintain mmap fd set & backup ipc_name
          const std::string &ipc_name = shared_writer_holder->ipc_name();
          memory::allocation::MemoryMapFdSet::Instance().Insert(ipc_name);
          // 5. copy data & reset holder
          memory::Copy(platform::CPUPlace(), shared_writer_holder->ptr(),
                       platform::CPUPlace(), data_ptr, data_size);
          t.ResetHolder(shared_writer_holder);
          // 6. append to result list
          tensors.append(t);
        }
        return tensors;
      },
      py::return_value_policy::take_ownership);

  m.def("_remove_tensor_list_mmap_fds", [](py::list &tensor_list) {
    for (size_t i = 0; i < tensor_list.size(); ++i) {
      auto t = tensor_list[i].cast<framework::LoDTensor>();
      auto *mmap_writer_allocation =
          dynamic_cast<memory::allocation::MemoryMapWriterAllocation *>(
              t.Holder().get());
      PADDLE_ENFORCE_NOT_NULL(
          mmap_writer_allocation,
          platform::errors::NotFound("The shared memory of LoDTensor in "
                                     "DataLoader's child process has been "
                                     "released."));
      memory::allocation::MemoryMapFdSet::Instance().Remove(
          mmap_writer_allocation->ipc_name());
    }
  });

  m.def("_cleanup_mmap_fds",
        []() { memory::allocation::MemoryMapFdSet::Instance().Clear(); });
#endif

305
  py::class_<imperative::detail::BackwardStrategy> backward_strategy(
306 307
      m, "BackwardStrategy", R"DOC(

J
Jiabin Yang 已提交
308
    BackwardStrategy is a descriptor of how to run the backward process.
309

J
Jiabin Yang 已提交
310
    **Note**:
T
tianshuo78520a 已提交
311
        **This API is only available in** `Dygraph <../../user_guides/howto/dygraph/DyGraph.html>`_ **Mode**
312

J
Jiabin Yang 已提交
313 314
    Attribute:
        **sort_sum_gradient**:
315

J
Jiabin Yang 已提交
316
        If framework will sum the gradient by the reverse order of trace. eg. x_var ( :ref:`api_guide_Variable` ) will be the input of multiple OP such as :ref:`api_fluid_layers_scale` , this attr will decide if framework will sum gradient of `x_var` by the reverse order.
L
lujun 已提交
317

J
Jiabin Yang 已提交
318
        By Default: False
L
lujun 已提交
319

J
Jiabin Yang 已提交
320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337
        Examples:
            .. code-block:: python

                import numpy as np
                import paddle.fluid as fluid

                x = np.ones([2, 2], np.float32)
                with fluid.dygraph.guard():
                    x_var = fluid.dygraph.to_variable(x)
                    sums_inputs = []
                    # x_var will be multi-scales' input here
                    for _ in range(10):
                        sums_inputs.append(fluid.layers.scale(x_var))
                    ret2 = fluid.layers.sums(sums_inputs)
                    loss2 = fluid.layers.reduce_sum(ret2)
                    backward_strategy = fluid.dygraph.BackwardStrategy()
                    backward_strategy.sort_sum_gradient = True
                    loss2.backward(backward_strategy)
338
      )DOC");
339 340 341 342 343 344 345 346 347 348 349 350 351 352 353
  backward_strategy.def(py::init())
      .def_property("sort_sum_gradient",
                    [](const imperative::detail::BackwardStrategy &self) {
                      return self.sorted_sum_gradient_;
                    },
                    [](imperative::detail::BackwardStrategy &self,
                       bool sorted_sum_gradient) {
                      self.sorted_sum_gradient_ = sorted_sum_gradient;
                    });

  m.def("start_imperative_gperf_profiler",
        []() { imperative::StartProfile(); });

  m.def("stop_imperative_gperf_profiler", []() { imperative::StopProfile(); });

Z
Zeng Jinle 已提交
354 355 356
  m.def("_is_dygraph_debug_enabled",
        []() { return imperative::IsDebugEnabled(); });
  m.def("_dygraph_debug_level", []() { return imperative::GetDebugLevel(); });
357 358 359 360
  m.def("_switch_tracer",
        [](const std::shared_ptr<imperative::Tracer> &tracer) {
          imperative::SetCurrentTracer(tracer);
        });
Z
Zeng Jinle 已提交
361

362
  py::class_<imperative::VarBase, std::shared_ptr<imperative::VarBase>>(
J
Jiabin Yang 已提交
363 364
      m, "VarBase",
      R"DOC()DOC")
Z
Zeng Jinle 已提交
365
      .def_static("_alive_vars", &imperative::VarBase::AliveVarNames)
J
Jiabin Yang 已提交
366
      .def("__init__",
367 368 369 370 371 372 373 374 375 376 377
           [](imperative::VarBase &self, framework::proto::VarType::Type dtype,
              const std::vector<int> &dims, const py::handle &name,
              framework::proto::VarType::Type type, bool persistable) {
             std::string act_name = "";
             if (!name.ptr() || name.ptr() == Py_None) {
               act_name = imperative::GetCurrentTracer()->GenerateUniqueName(
                   "generated_var");
             } else {
               act_name = name.cast<std::string>();
             }
             new (&self) imperative::VarBase(act_name);
J
Jiabin Yang 已提交
378 379 380 381 382 383 384 385 386
             self.SetPersistable(persistable);
             self.SetType(type);
             self.SetDataType(dtype);
             if (type == framework::proto::VarType::LOD_TENSOR) {
               auto *tensor =
                   self.MutableVar()->GetMutable<framework::LoDTensor>();
               tensor->Resize(framework::make_ddim(dims));
             }
           })
387 388
      .def("__init__", &InitVarBaseFromNumpyWithArg<platform::CPUPlace>,
           py::arg("value"), py::arg("place"), py::arg("persistable") = false,
L
Leo Chen 已提交
389
           py::arg("zero_copy") = false, py::arg("name") = "")
390 391
      .def("__init__", &InitVarBaseFromNumpyWithArg<platform::CUDAPlace>,
           py::arg("value"), py::arg("place"), py::arg("persistable") = false,
L
Leo Chen 已提交
392
           py::arg("zero_copy") = false, py::arg("name") = "")
393 394
      .def("__init__", &InitVarBaseFromNumpyWithArg<platform::CUDAPinnedPlace>,
           py::arg("value"), py::arg("place"), py::arg("persistable") = false,
L
Leo Chen 已提交
395 396
           py::arg("zero_copy") = false, py::arg("name") = "")
      .def("__init__", &InitVarBaseFromNumpyWithArgDefault, py::arg("value"))
397
      .def("__init__", &InitVarBaseFromNumpyWithKwargs)
398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491
      .def("__getitem__",
           [](imperative::VarBase &self, py::handle _index) {
             // We allow indexing by Integers, Slices, and tuples of those
             // types.
             // Ellipsis and None are not supported yet.
             std::vector<int> slice_axes, slice_starts, slice_ends,
                 slice_strides, decrease_axis;
             // wrap to tuple
             PyObject *index = !PyTuple_Check(_index.ptr())
                                   ? PyTuple_Pack(1, _index.ptr())
                                   : _index.ptr();
             const auto &tensor = self.Var().Get<framework::LoDTensor>();
             PADDLE_ENFORCE_EQ(tensor.IsInitialized(), true,
                               platform::errors::InvalidArgument(
                                   "%s has not been initialized", self.Name()));
             const auto &shape = tensor.dims();
             const int rank = shape.size();
             const int size = PyTuple_GET_SIZE(index);
             PADDLE_ENFORCE_EQ(
                 size <= rank, true,
                 platform::errors::InvalidArgument(
                     "too many indices (%d) for tensor of dimension %d", size,
                     rank));
             for (int dim = 0; dim < size; ++dim) {
               PyObject *slice_item = PyTuple_GetItem(index, dim);
               PADDLE_ENFORCE_EQ(
                   PyNumber_Check(slice_item) || PySlice_Check(slice_item),
                   true,
                   platform::errors::InvalidArgument(
                       "We allow indexing by Integers, Slices, and tuples of "
                       "these types, but received %s in %dth slice item",
                       std::string(Py_TYPE(slice_item)->tp_name), dim + 1));
               int dim_len = shape[dim];
               if (PyNumber_Check(slice_item)) {
                 // integer
                 int start = static_cast<int>(PyLong_AsLong(slice_item));
                 start = start < 0 ? start + dim_len : start;
                 slice_axes.push_back(dim);
                 slice_starts.push_back(start);
                 slice_ends.push_back(start + 1);
                 slice_strides.push_back(1);
                 decrease_axis.push_back(dim);
               } else {
                 // slice
                 Py_ssize_t start, end, step;
// The parameter type for the slice parameter was PySliceObject* before 3.2
#if PY_VERSION_HEX >= 0x03020000
                 PySlice_GetIndices(slice_item, dim_len, &start, &end, &step);
#else
                 PySlice_GetIndices(
                     reinterpret_cast<PySliceObject *>(slice_item), dim_len,
                     &start, &end, &step);
#endif
                 // :: or : or 0:dim_len:1
                 if (start == 0 && end == dim_len && step == 1) continue;
                 slice_axes.push_back(dim);
                 slice_starts.push_back(start);
                 slice_ends.push_back(end);
                 slice_strides.push_back(step);
               }
             }
             if (!PyTuple_Check(_index.ptr())) Py_DecRef(index);

             // release gil and do tracing
             py::gil_scoped_release release;
             const auto &tracer = imperative::GetCurrentTracer();
             auto _self = self.NewVarBase(tensor.place(), false);
             if (slice_axes.empty()) {
               return _self;
             } else {
               std::vector<int> infer_flags(size, 1);
               imperative::NameVarBaseMap ins = {{"Input", {_self}}};
               framework::AttributeMap attrs = {
                   {"axes", slice_axes},
                   {"starts", slice_starts},
                   {"ends", slice_ends},
                   {"infer_flags", infer_flags},
                   {"decrease_axis", decrease_axis}};
               auto out = std::shared_ptr<imperative::VarBase>(
                   new imperative::VarBase(tracer->GenerateUniqueName()));
               imperative::NameVarBaseMap outs = {{"Out", {out}}};
               std::string op_type = "slice";
               for (auto stride : slice_strides) {
                 if (stride != 1) {
                   op_type = "strided_slice";
                   attrs.insert({"strides", slice_strides});
                   attrs.erase("decrease_axis");
                   break;
                 }
               }
               tracer->TraceOp(op_type, ins, outs, std::move(attrs));
               return out;
             }
           })
492 493 494 495 496 497 498 499 500 501 502 503 504
      .def("numpy",
           [](imperative::VarBase &self) -> py::array {
             const auto &tensor =
                 self.MutableVar()->Get<framework::LoDTensor>();
             PADDLE_ENFORCE_EQ(
                 tensor.IsInitialized(), true,
                 platform::errors::InvalidArgument(
                     "%s is Empty, Please check if it has no data in",
                     self.Name()));
             return TensorToPyArray(tensor, true);
           },
           R"DOC(
        **Notes**:
T
tianshuo78520a 已提交
505
            **This API is ONLY available in Dygraph mode**
506 507 508 509 510 511 512 513 514 515 516 517 518 519

        Returns a numpy array shows the value of current :ref:`api_guide_Variable_en`

        Returns:
            ndarray: The numpy value of current Variable.

        Returns type:
            ndarray: dtype is same as current Variable

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid
                from paddle.fluid.dygraph.base import to_variable
520
                from paddle.fluid.dygraph import Linear
521 522 523 524
                import numpy as np

                data = np.random.uniform(-1, 1, [30, 10, 32]).astype('float32')
                with fluid.dygraph.guard():
525
                    linear = Linear(32, 64)
526
                    data = to_variable(data)
527
                    x = linear(data)
528 529 530 531 532 533 534 535 536 537 538 539 540
                    print(x.numpy())

       )DOC")
      .def("detach",
           [](const imperative::VarBase &self) {
             const auto &tensor = self.Var().Get<framework::LoDTensor>();
             PADDLE_ENFORCE_EQ(tensor.IsInitialized(), true,
                               platform::errors::InvalidArgument(
                                   "%s has not been initialized", self.Name()));
             return self.NewVarBase(tensor.place(), false);
           },
           py::return_value_policy::copy, R"DOC(
        **Notes**:
T
tianshuo78520a 已提交
541
            **This API is ONLY available in Dygraph mode**
542 543 544 545 546 547 548 549 550 551 552 553

        Returns a new Variable, detached from the current graph.

        Returns:
             ( :ref:`api_guide_Variable_en` | dtype is same as current Variable): The detached Variable.


        Examples:
            .. code-block:: python

                import paddle.fluid as fluid
                from paddle.fluid.dygraph.base import to_variable
554
                from paddle.fluid.dygraph import Linear
555 556 557 558
                import numpy as np

                data = np.random.uniform(-1, 1, [30, 10, 32]).astype('float32')
                with fluid.dygraph.guard():
559
                    linear = Linear(32, 64)
560
                    data = to_variable(data)
561
                    x = linear(data)
562 563 564 565 566 567
                    y = x.detach()

       )DOC")
      .def("clear_gradient", &imperative::VarBase::ClearGradient, R"DOC(

        **Notes**:
T
tianshuo78520a 已提交
568
        **1. This API is ONLY available in Dygraph mode**
569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597

        **2. Use it only Variable has gradient, normally we use this for Parameters since other temporal Variable will be deleted by Python's GC**

        Clear  (set to ``0`` ) the Gradient of Current Variable

        Returns:  None

        Examples:
             .. code-block:: python

                import paddle.fluid as fluid
                import numpy as np

                x = np.ones([2, 2], np.float32)
                with fluid.dygraph.guard():
                    inputs2 = []
                    for _ in range(10):
                         tmp = fluid.dygraph.base.to_variable(x)
                         tmp.stop_gradient=False
                         inputs2.append(tmp)
                    ret2 = fluid.layers.sums(inputs2)
                    loss2 = fluid.layers.reduce_sum(ret2)
                    backward_strategy = fluid.dygraph.BackwardStrategy()
                    backward_strategy.sort_sum_gradient = True
                    loss2.backward(backward_strategy)
                    print(loss2.gradient())
                    loss2.clear_gradient()
                    print("After clear {}".format(loss2.gradient()))
      )DOC")
L
Leo Chen 已提交
598 599 600 601 602 603
      .def("_run_backward",
           [](imperative::VarBase &self,
              const imperative::detail::BackwardStrategy &bckst,
              const imperative::Tracer &tracer) {
             // TODO(jiabin): when we impl more backward execution we can select
             // them
604
             auto *engine = tracer.GetEngine();
L
Leo Chen 已提交
605
             engine->Init(&self, bckst);
606
             VLOG(3) << "Start backward";
L
Leo Chen 已提交
607 608 609 610 611 612 613 614 615 616
             engine->Execute();
             VLOG(3) << "Finish backward";
           },
           py::call_guard<py::gil_scoped_release>())
      .def("_grad_name", &imperative::VarBase::GradVarName)
      .def("_grad_value",
           [](imperative::VarBase &self) {
             return self.MutableGradVar()->Get<framework::LoDTensor>();
           },
           py::return_value_policy::reference)
617
      .def("_grad_ivar",
J
Jiabin Yang 已提交
618 619
           [](const imperative::VarBase &self) {
             auto &grad_var = self.GradVarBase();
620 621 622 623 624 625 626 627 628 629 630
             if (grad_var && grad_var->Var().IsInitialized()) {
               auto *tensor =
                   grad_var->MutableVar()->IsType<framework::LoDTensor>()
                       ? grad_var->MutableVar()
                             ->GetMutable<framework::LoDTensor>()
                       : grad_var->MutableVar()
                             ->GetMutable<framework::SelectedRows>()
                             ->mutable_value();
               if (tensor->IsInitialized()) {
                 return grad_var;
               }
J
Jiabin Yang 已提交
631
             }
632
             return std::shared_ptr<imperative::VarBase>(nullptr);
J
Jiabin Yang 已提交
633 634
           },
           py::return_value_policy::copy)
635 636
      .def("_copy_to",
           [](const imperative::VarBase &self, const platform::CPUPlace &place,
J
Jiabin Yang 已提交
637 638
              bool blocking) { return self.NewVarBase(place, blocking); },
           py::return_value_policy::copy)
639 640
      .def("_copy_to",
           [](const imperative::VarBase &self, const platform::CUDAPlace &place,
J
Jiabin Yang 已提交
641 642 643
              bool blocking) { return self.NewVarBase(place, blocking); },
           py::return_value_policy::copy)
      .def("value", [](imperative::VarBase &self) { return self.MutableVar(); },
644 645 646
           py::return_value_policy::reference)
      .def_property("name", &imperative::VarBase::Name,
                    &imperative::VarBase::SetName)
L
Leo Chen 已提交
647 648 649 650 651
      .def_property("stop_gradient",
                    &imperative::VarBase::OverridedStopGradient,
                    &imperative::VarBase::SetOverridedStopGradient)
      .def_property("persistable", &imperative::VarBase::Persistable,
                    &imperative::VarBase::SetPersistable)
J
Jiabin Yang 已提交
652 653 654 655
      .def_property_readonly(
          "shape",
          [](imperative::VarBase &self) {
            if (self.Var().IsType<framework::LoDTensor>()) {
656
              return framework::vectorize<int>(
J
Jiabin Yang 已提交
657
                  self.Var().Get<framework::LoDTensor>().dims());
658 659 660
            } else if (self.Var().IsType<framework::SelectedRows>()) {
              return framework::vectorize<int>(
                  self.Var().Get<framework::SelectedRows>().value().dims());
J
Jiabin Yang 已提交
661 662 663 664 665 666 667
            } else {
              VLOG(2) << "It is meaningless to get shape of variable type "
                      << GetTypeName(self);
              return std::vector<int>();
            }
          })
      .def_property_readonly("type", &imperative::VarBase::Type)
L
Leo Chen 已提交
668
      .def_property_readonly("dtype", &imperative::VarBase::DataType);
669 670 671

  py::class_<imperative::Layer, Layer /* <--- trampoline*/> layer(m, "Layer");
  layer.def(py::init<>())
672 673 674 675 676
      .def("forward",
           [](imperative::Layer &self,
              const std::vector<std::shared_ptr<imperative::VarBase>> &inputs) {
             return self.Forward(inputs);
           });
677

678 679 680 681 682
  py::class_<imperative::jit::ProgramDescTracer>(m, "ProgramDescTracer", "")
      .def("create_program_desc",
           &imperative::jit::ProgramDescTracer::CreateProgramDesc)
      .def("reset", &imperative::jit::ProgramDescTracer::Reset);

683 684 685
  py::class_<imperative::Tracer, std::shared_ptr<imperative::Tracer>>(
      m, "Tracer",
      R"DOC()DOC")
686
      .def("__init__",
J
Jiabin Yang 已提交
687
           [](imperative::Tracer &self) { new (&self) imperative::Tracer(); })
688 689 690
      .def_property("_enable_program_desc_tracing",
                    &imperative::Tracer::IsProgramDescTracingEnabled,
                    &imperative::Tracer::SetEnableProgramDescTracing)
691 692 693 694 695 696 697 698 699 700
      .def_property("_train_mode", &imperative::Tracer::NoGrad,
                    &imperative::Tracer::SetNoGrad)
      .def_property(
          "_expected_place",
          [](const imperative::Tracer &self) -> py::object {
            return py::cast(self.ExpectedPlace());
          },
          [](imperative::Tracer &self, const py::object &obj) {
            if (py::isinstance<platform::CUDAPlace>(obj)) {
              auto p = obj.cast<platform::CUDAPlace *>();
L
Leo Chen 已提交
701
              self.SetExpectedPlace(*p);
702 703
            } else if (py::isinstance<platform::CPUPlace>(obj)) {
              auto p = obj.cast<platform::CPUPlace *>();
L
Leo Chen 已提交
704
              self.SetExpectedPlace(*p);
705 706
            } else if (py::isinstance<platform::CUDAPinnedPlace>(obj)) {
              auto p = obj.cast<platform::CUDAPinnedPlace *>();
L
Leo Chen 已提交
707
              self.SetExpectedPlace(*p);
708
            } else {
L
Leo Chen 已提交
709
              PADDLE_THROW(platform::errors::InvalidArgument(
710
                  "Incompatible Place Type: supports CUDAPlace, CPUPlace, "
L
Leo Chen 已提交
711 712
                  "and CUDAPinnedPlace, "
                  "but got Unknown Type!"));
713 714
            }
          })
715 716 717
      .def("_get_program_desc_tracer",
           &imperative::Tracer::GetProgramDescTracer,
           py::return_value_policy::reference)
M
minqiyang 已提交
718
      .def("trace",
J
Jiabin Yang 已提交
719 720 721 722 723 724
           [](imperative::Tracer &self, const std::string &type,
              const PyNameVarBaseMap &ins, const PyNameVarBaseMap &outs,
              framework::AttributeMap attrs, const platform::CUDAPlace &place,
              bool trace_backward) {
             auto ins_map = ConvertToNameVarBaseMap(ins);
             auto outs_map = ConvertToNameVarBaseMap(outs);
725 726
             {
               py::gil_scoped_release release;
J
Jiabin Yang 已提交
727 728
               self.TraceOp(type, std::move(ins_map), std::move(outs_map),
                            std::move(attrs), place, trace_backward);
729
             }
M
minqiyang 已提交
730
           })
J
Jiabin Yang 已提交
731 732 733 734 735 736 737 738 739 740 741 742 743
      .def("trace",
           [](imperative::Tracer &self, const std::string &type,
              const PyNameVarBaseMap &ins, const PyNameVarBaseMap &outs,
              framework::AttributeMap attrs, const platform::CPUPlace &place,
              bool trace_backward) {
             auto ins_map = ConvertToNameVarBaseMap(ins);
             auto outs_map = ConvertToNameVarBaseMap(outs);
             {
               py::gil_scoped_release release;
               self.TraceOp(type, std::move(ins_map), std::move(outs_map),
                            std::move(attrs), place, trace_backward);
             }
           });
744 745

  // define parallel context
746 747 748
  py::class_<imperative::ParallelStrategy> parallel_strategy(
      m, "ParallelStrategy", "");
  parallel_strategy.def(py::init())
749 750
      .def_property(
          "nranks",
751 752
          [](const imperative::ParallelStrategy &self) { return self.nranks_; },
          [](imperative::ParallelStrategy &self, int nranks) {
753 754 755
            self.nranks_ = nranks;
          })
      .def_property("local_rank",
756
                    [](const imperative::ParallelStrategy &self) {
757 758
                      return self.local_rank_;
                    },
759
                    [](imperative::ParallelStrategy &self, int local_rank) {
760 761 762 763
                      self.local_rank_ = local_rank;
                    })
      .def_property(
          "trainer_endpoints",
764
          [](const imperative::ParallelStrategy &self) {
765 766
            return self.trainer_endpoints_;
          },
767
          [](imperative::ParallelStrategy &self, std::vector<std::string> eps) {
768 769 770
            self.trainer_endpoints_ = eps;
          })
      .def_property("current_endpoint",
771
                    [](const imperative::ParallelStrategy &self) {
772 773
                      return self.current_endpoint_;
                    },
774 775
                    [](imperative::ParallelStrategy &self,
                       const std::string &ep) { self.current_endpoint_ = ep; });
776 777 778 779 780 781 782 783 784 785

  m.def(
      "dygraph_partial_grad",
      [](const std::vector<std::shared_ptr<imperative::VarBase>> &input_targets,
         const std::vector<std::shared_ptr<imperative::VarBase>>
             &output_targets,
         const std::vector<std::shared_ptr<imperative::VarBase>> &output_grads,
         const std::vector<std::shared_ptr<imperative::VarBase>> &no_grad_vars,
         const platform::Place &place,
         const imperative::detail::BackwardStrategy &strategy,
Z
Zeng Jinle 已提交
786 787 788 789 790
         bool create_graph, bool retain_graph, bool allow_unused,
         bool only_inputs) {
        imperative::PartialGradEngine engine(
            input_targets, output_targets, output_grads, no_grad_vars, place,
            strategy, create_graph, retain_graph, allow_unused, only_inputs);
791 792 793 794 795
        engine.Execute();
        return engine.GetResult();
      },
      py::call_guard<py::gil_scoped_release>());

796
#if defined(PADDLE_WITH_NCCL)
797 798
  py::class_<imperative::NCCLParallelContext> nccl_ctx(m,
                                                       "NCCLParallelContext");
799 800

  nccl_ctx
801 802 803
      .def(py::init<const imperative::ParallelStrategy &,
                    const platform::CUDAPlace &>())
      .def("init", [](imperative::NCCLParallelContext &self) { self.Init(); });
804
#endif
805 806 807 808
}

}  // namespace pybind
}  // namespace paddle