diff --git a/paddle/fluid/eager/eager_tensor.h b/paddle/fluid/eager/eager_tensor.h index b175f07cab4b7c957f4285ce5ff55181cc01afc4..1a3a5a7f9ee0aa1478118bc37a423931e2b9e252 100644 --- a/paddle/fluid/eager/eager_tensor.h +++ b/paddle/fluid/eager/eager_tensor.h @@ -22,6 +22,7 @@ #include "paddle/pten/api/all.h" #include "paddle/pten/api/lib/api_declare.h" #include "paddle/pten/api/lib/utils/tensor_utils.h" +#include "paddle/pten/core/convert_utils.h" /** * This class is used by Eager mode for now. It's painful to do this in Eager * Mode, the better @@ -57,6 +58,7 @@ class EagerTensor final { explicit EagerTensor(const std::shared_ptr& tensor_impl) : tensor_(std::make_shared(tensor_impl)), var_(paddle::framework::Variable()) {} + EagerTensor(const EagerTensor&) = default; EagerTensor(EagerTensor&&) = default; @@ -163,6 +165,37 @@ class EagerTensor final { */ void reset() { tensor_->reset(); } + /** + * @brief Transfer the current Tensor to the specified device and return. + * + * @param place, the target place of which the tensor will copy to. + * @return Tensor + */ + // TODO(chenweihang): replace Backend by new Place + EagerTensor copy_to(pten::Backend backend, bool blocking) const { + if (Var().IsInitialized()) { + const_cast(this)->SyncToTensor(); + } + return EagerTensor(tensor_->copy_to(backend, blocking)); + } + + /** + * @brief Transfer the source Tensor to current Tensor. + * + * @param src, the source Tensor to be copied. + * @param blocking, Should we copy this in sync way. + * @return void + */ + void copy_(const EagerTensor& src, const bool blocking) { + if (src.Var().IsInitialized()) { + const_cast(&src)->SyncToTensor(); + } + if (Var().IsInitialized()) { + SyncToTensor(); + } + tensor_->copy_(*(src.tensor_.get()), blocking); + } + /* Part 6: Operator overloading */ EagerTensor& operator=(const EagerTensor& x) & { tensor_ = x.tensor_; @@ -270,6 +303,16 @@ class EagerTensor final { } private: + /** + * @description: Use a pten::Tensor pointer to construct a EagerTensor, never + * public this!!!!. + * @param {pten::Tensor} tensor + * @return {EagerTensor} + */ + explicit EagerTensor(const paddle::experimental::Tensor& tensor) + : tensor_(std::make_shared(tensor)), + var_(paddle::framework::Variable()) {} + std::shared_ptr tensor_ = nullptr; paddle::framework::Variable var_; }; diff --git a/paddle/fluid/pybind/eager.cc b/paddle/fluid/pybind/eager.cc index c500e9fbb9ee64c7c0b94b96ec51187066380a7c..94ff2eb4c1e231e7dba7655d344431f6c402c363 100644 --- a/paddle/fluid/pybind/eager.cc +++ b/paddle/fluid/pybind/eager.cc @@ -26,15 +26,21 @@ limitations under the License. */ #include "paddle/pten/core/convert_utils.h" #include "paddle/pten/core/dense_tensor.h" #include "paddle/pten/include/core.h" +#include "pybind11/numpy.h" +#include "pybind11/pybind11.h" #pragma GCC diagnostic ignored "-Wmissing-field-initializers" +#include "paddle/fluid/framework/python_headers.h" #include "paddle/fluid/pybind/eager_op_function_impl.h" - +#include "paddle/fluid/pybind/tensor_py.h" +#include "paddle/pten/api/lib/utils/storage.h" +#include "paddle/pten/api/lib/utils/tensor_utils.h" namespace paddle { namespace pybind { namespace py = ::pybind11; PyTypeObject* p_eager_tensor_type; +extern PyTypeObject* g_vartype_pytype; PyObject* EagerTensorNew(PyTypeObject* type, PyObject* args, PyObject* kwargs) { PyObject* obj = type->tp_alloc(type, 0); @@ -45,6 +51,401 @@ PyObject* EagerTensorNew(PyTypeObject* type, PyObject* args, PyObject* kwargs) { return obj; } +// TODO(jiabin): Overload this once we need more constructor in Python +void EmptyEagerTensorInitializer( + EagerTensorObject* self, const std::string& name, + const paddle::platform::Place& place, bool persistable = false, + bool stop_gradient = true, framework::proto::VarType::Type dtype = + paddle::framework::proto::VarType::FP32, + const std::vector& dims = {}, + framework::proto::VarType::Type var_type = + paddle::framework::proto::VarType::LOD_TENSOR) { + self->eager_tensor.set_name(name); + auto autograd_meta = egr::EagerUtils::autograd_meta(&(self->eager_tensor)); + autograd_meta->SetPersistable(persistable); + autograd_meta->SetStopGradient(stop_gradient); + if (var_type == paddle::framework::proto::VarType::LOD_TENSOR) { + // TODO(jiabin): Maybe support LOD later + std::shared_ptr dense_tensor = + std::make_shared( + pten::make_intrusive(place), + pten::DenseTensorMeta(pten::TransToPtenDataType(dtype), + paddle::framework::make_ddim(dims))); + self->eager_tensor.set_impl(dense_tensor); + } +} + +void InitEagerTensorWithNumpyValue(EagerTensorObject* self, + const py::object& array, + bool zero_copy = false) { + PADDLE_ENFORCE_EQ( + self->eager_tensor.defined(), true, + paddle::platform::errors::Fatal( + "Calling InitEagerTensorWithNumpyValue of Eager Tensor without " + "EmptyEagerTensorInitializer is " + "forbidden. Please check your code and make sure you new a " + "eager tensor before init it with NumPy.")); + pten::DenseTensor* impl_ptr = + static_cast(self->eager_tensor.impl().get()); + paddle::platform::Place place = impl_ptr->place(); + paddle::framework::LoDTensor temp_tensor = paddle::framework::LoDTensor(); + if (platform::is_cpu_place(place)) { + SetTensorFromPyArray( + &temp_tensor, array, BOOST_GET_CONST(platform::CPUPlace, place), + zero_copy); + } else if (platform::is_xpu_place(place)) { + SetTensorFromPyArray( + &temp_tensor, array, BOOST_GET_CONST(platform::XPUPlace, place), + zero_copy); + } else if (platform::is_gpu_place(place)) { + SetTensorFromPyArray( + &temp_tensor, array, BOOST_GET_CONST(platform::CUDAPlace, place), + zero_copy); + } else if (platform::is_cuda_pinned_place(place)) { + SetTensorFromPyArray( + &temp_tensor, array, BOOST_GET_CONST(platform::CUDAPinnedPlace, place), + zero_copy); + } else if (platform::is_npu_place(place)) { + SetTensorFromPyArray( + &temp_tensor, array, BOOST_GET_CONST(platform::NPUPlace, place), + zero_copy); + } else { + PADDLE_THROW(platform::errors::InvalidArgument( + "Place should be one of " + "CPUPlace/XPUPlace/CUDAPlace/CUDAPinnedPlace/NPUPlace")); + } + paddle::experimental::ReMakePtenDenseTensor(temp_tensor, impl_ptr); +} + +void InitEagerTensorWithEagerTensor(EagerTensorObject* self, + const egr::EagerTensor& src, + const paddle::platform::Place& place, + const std::string& name) { + self->eager_tensor.set_name(name); + if (place == src.place()) { + auto impl = std::static_pointer_cast(src.impl()); + self->eager_tensor.set_impl(impl); + VLOG(4) << "Same place, do ShareDataWith"; + } else { + self->eager_tensor.set_impl( + src.copy_to(pten::TransToPtenBackend(place), true).impl()); + VLOG(4) << "Different place, do TensorCopy"; + } + egr::EagerUtils::autograd_meta(&(self->eager_tensor))->SetStopGradient(true); + if (src.get_autograd_meta()) { + egr::EagerUtils::unsafe_autograd_meta(self->eager_tensor) + ->SetPersistable( + egr::EagerUtils::unsafe_autograd_meta(src)->Persistable()); + } else { + egr::EagerUtils::unsafe_autograd_meta(self->eager_tensor) + ->SetPersistable(false); + } +} + +// TODO(jiabin): We have to do some ugly work, refactor this method using +// PyArg_ParseTuple()，PyArg_ParseTupleAndKeywords() and PyArg_Parse() later to +// support kwargs. +int EagerTensorInit(PyObject* self, PyObject* args, PyObject* kwds) { + /** We should have init function with signature: + * 1. + * def __init__ () + * 2. + * def __init__ ( + * ** dtype: paddle::framework::proto::VarType::Type, + * ** dims: vector, + * ** name: std::string, + * ** type: paddle::framework::proto::VarType::Type, + * ** persistable: bool) + * 3. (multi-place) (must have first 2 parameter) + * def __init__ ( + * ** value: ndarray, + * ** place: paddle::platform::Place, + * ** persistable: bool, + * ** zero_copy: bool, + * ** name: std::string, + * ** stop_gradient: bool) + * 4. + * def __init__ ( + * ** value: ndarray) + * 5. + * def __init__ ( + * ** tensor: EagerTensor) + * 6. (multi-place) (must have first 2 parameter) + * def __init__ ( + * ** tensor: EagerTensor, + * ** place: paddle::platform::Place, + * ** name: std::string) + * **/ + PADDLE_ENFORCE_NOT_NULL( + self, paddle::platform::errors::Fatal( + "Calling __init__ of Eager Tensor without __new__ is " + "forbidden. Please check your code and make sure you new a " + "eager tensor before init it.")); + + auto py_tensor_ptr = reinterpret_cast(self); + + // TODO(jiabin): Only support case 2 for now + Py_ssize_t args_num = PyTuple_Size(args); + switch (args_num) { + case (Py_ssize_t)0: { + // case 1 + VLOG(6) << "Calling case1's initializer."; + EmptyEagerTensorInitializer( + py_tensor_ptr, + egr::Controller::Instance().GenerateUniqueName("generated_tensor"), + egr::Controller::Instance().GetExpectedPlace()); + return 0; + } + case (Py_ssize_t)1: { + // case 4, 5 + PyObject* arg0_ptr = PyTuple_GET_ITEM(args, 0); + if (pybind11::detail::npy_api::get().PyArray_Check_(arg0_ptr)) { + VLOG(6) << "Calling case4's initializer."; + PADDLE_ENFORCE_EQ( + pybind11::detail::npy_api::get().PyArray_Check_(arg0_ptr), true, + paddle::platform::errors::Fatal( + "We expected initial parametes list like: \n **value: ndarray. " + "But got value with wrong type: %s", + reinterpret_cast(arg0_ptr->ob_type)->tp_name)); + py::object numpy_value = py::object(py::handle(arg0_ptr), true); + EmptyEagerTensorInitializer( + py_tensor_ptr, + egr::Controller::Instance().GenerateUniqueName("generated_tensor"), + egr::Controller::Instance().GetExpectedPlace()); + InitEagerTensorWithNumpyValue(py_tensor_ptr, numpy_value, + /** zero copy **/ false); + return 0; + } else if (PyObject_IsInstance(arg0_ptr, reinterpret_cast( + p_eager_tensor_type))) { + VLOG(6) << "Calling case5's initializer."; + auto src_tensor = CastPyArg2EagerTensor(arg0_ptr, 0); + InitEagerTensorWithEagerTensor( + py_tensor_ptr, src_tensor, + egr::Controller::Instance().GetExpectedPlace(), + egr::Controller::Instance().GenerateUniqueName("generated_tensor")); + return 0; + } else { + PADDLE_THROW(platform::errors::InvalidArgument( + "We only support construct tensor from numpy value or tensor with " + "python args by this initializer, " + "please check your input first and make sure you are on the right " + "way.")); + } + return 0; + } + case (Py_ssize_t)2: { + PyObject* arg0_ptr = PyTuple_GET_ITEM(args, 0); + if (pybind11::detail::npy_api::get().PyArray_Check_(arg0_ptr)) { + VLOG(6) << "Calling case3's initializer."; + PADDLE_ENFORCE_EQ( + pybind11::detail::npy_api::get().PyArray_Check_(arg0_ptr), true, + paddle::platform::errors::Fatal( + "We expected initial parametes list like: \n **value: ndarray. " + "But got value with wrong type: %s", + reinterpret_cast(arg0_ptr->ob_type)->tp_name)); + py::object numpy_value = py::object(py::handle(arg0_ptr), true); + paddle::platform::Place place = + CastPyArg2Place(PyTuple_GET_ITEM(args, 1), 1); + EmptyEagerTensorInitializer( + py_tensor_ptr, + egr::Controller::Instance().GenerateUniqueName("generated_tensor"), + place); + InitEagerTensorWithNumpyValue(py_tensor_ptr, numpy_value, + /** zero copy **/ false); + return 0; + } else if (PyObject_IsInstance(arg0_ptr, reinterpret_cast( + p_eager_tensor_type))) { + VLOG(6) << "Calling case6's initializer."; + auto src_tensor = CastPyArg2EagerTensor(arg0_ptr, 0); + paddle::platform::Place place = + CastPyArg2Place(PyTuple_GET_ITEM(args, 1), 1); + InitEagerTensorWithEagerTensor( + py_tensor_ptr, src_tensor, place, + egr::Controller::Instance().GenerateUniqueName("generated_tensor")); + return 0; + } else { + PADDLE_THROW(platform::errors::InvalidArgument( + "We only support construct tensor from numpy value or tensor with " + "python args by this initializer, " + "please check your input first and make sure you are on the right " + "way.")); + } + } + case (Py_ssize_t)3: { + PyObject* arg0_ptr = PyTuple_GET_ITEM(args, 0); + if (pybind11::detail::npy_api::get().PyArray_Check_(arg0_ptr)) { + VLOG(6) << "Calling case3's initializer."; + PADDLE_ENFORCE_EQ( + pybind11::detail::npy_api::get().PyArray_Check_(arg0_ptr), true, + paddle::platform::errors::Fatal( + "We expected initial parametes list like: \n **value: ndarray. " + "But got value with wrong type: %s", + reinterpret_cast(arg0_ptr->ob_type)->tp_name)); + py::object numpy_value = py::object(py::handle(arg0_ptr), true); + paddle::platform::Place place = + CastPyArg2Place(PyTuple_GET_ITEM(args, 1), 1); + bool persistable = CastPyArg2AttrBoolean(PyTuple_GET_ITEM(args, 2), 2); + EmptyEagerTensorInitializer( + py_tensor_ptr, + egr::Controller::Instance().GenerateUniqueName("generated_tensor"), + place, persistable); + InitEagerTensorWithNumpyValue(py_tensor_ptr, numpy_value, + /** zero copy **/ false); + return 0; + } else if (PyObject_IsInstance(arg0_ptr, reinterpret_cast( + p_eager_tensor_type))) { + VLOG(6) << "Calling case6's initializer."; + auto src_tensor = CastPyArg2EagerTensor(arg0_ptr, 0); + paddle::platform::Place place = + CastPyArg2Place(PyTuple_GET_ITEM(args, 1), 1); + std::string act_name = ""; + PyObject* name_obj = PyTuple_GET_ITEM(args, 2); + if (name_obj == Py_None) { + act_name = egr::Controller::Instance().GenerateUniqueName( + "generated_tensor"); + } else { + act_name = CastPyArg2AttrString(name_obj, 2); + } + InitEagerTensorWithEagerTensor(py_tensor_ptr, src_tensor, place, + act_name); + return 0; + } else { + PADDLE_THROW(platform::errors::InvalidArgument( + "We only support construct tensor from numpy value or tensor with " + "python args by this initializer, " + "please check your input first and make sure you are on the right " + "way.")); + } + } + case (Py_ssize_t)4: { + VLOG(6) << "Calling case3's initializer."; + PyObject* arg0_ptr = PyTuple_GET_ITEM(args, 0); + PADDLE_ENFORCE_EQ( + pybind11::detail::npy_api::get().PyArray_Check_(arg0_ptr), true, + paddle::platform::errors::Fatal( + "We expected initial parametes list like: \n **value: ndarray, " + "\n ** place: paddle::platform::Place, \n ** persistable: bool, " + "\n ** zero_copy: bool, \n ** name: std::string, \n ** " + "stop_gradient: bool. But got value with wrong type: %s", + reinterpret_cast(arg0_ptr->ob_type)->tp_name)); + py::object numpy_value = + py::object(py::handle(PyTuple_GET_ITEM(args, 0)), true); + paddle::platform::Place place = + CastPyArg2Place(PyTuple_GET_ITEM(args, 1), 1); + bool persistable = CastPyArg2AttrBoolean(PyTuple_GET_ITEM(args, 2), 2); + bool zero_copy = CastPyArg2AttrBoolean(PyTuple_GET_ITEM(args, 3), 3); + EmptyEagerTensorInitializer( + py_tensor_ptr, + egr::Controller::Instance().GenerateUniqueName("generated_tensor"), + place, persistable); + InitEagerTensorWithNumpyValue(py_tensor_ptr, numpy_value, zero_copy); + return 0; + } + case (Py_ssize_t)5: { + // case 2 + PyObject* arg0_ptr = PyTuple_GET_ITEM(args, 0); + if (PyObject_IsInstance(arg0_ptr, + reinterpret_cast(g_vartype_pytype))) { + VLOG(6) << "Calling case2's initializer."; + paddle::framework::proto::VarType::Type dtype = + CastPyArg2ProtoType(PyTuple_GET_ITEM(args, 0), 0); + std::vector dims = + + CastPyArg2VectorOfInt(PyTuple_GET_ITEM(args, 1), 1); + std::string act_name = ""; + PyObject* name_obj = PyTuple_GET_ITEM(args, 2); + if (name_obj == Py_None) { + act_name = egr::Controller::Instance().GenerateUniqueName( + "generated_tensor"); + } else { + act_name = CastPyArg2AttrString(PyTuple_GET_ITEM(args, 2), 2); + } + paddle::framework::proto::VarType::Type var_type = + CastPyArg2ProtoType(PyTuple_GET_ITEM(args, 3), 3); + bool persistable = CastPyArg2AttrBoolean(PyTuple_GET_ITEM(args, 4), 4); + EmptyEagerTensorInitializer( + py_tensor_ptr, act_name, + egr::Controller::Instance().GetExpectedPlace(), persistable, true, + dtype, dims, var_type); + return 0; + } else if (PyObject_IsInstance(arg0_ptr, reinterpret_cast( + p_eager_tensor_type))) { + PADDLE_ENFORCE_EQ( + pybind11::detail::npy_api::get().PyArray_Check_(arg0_ptr), true, + paddle::platform::errors::Fatal( + "We expected initial parametes list like: \n **value: ndarray, " + "\n ** place: paddle::platform::Place, \n ** persistable: " + "bool, \n ** zero_copy: bool, \n ** name: std::string, \n ** " + "stop_gradient: bool. But got value with wrong type: %s", + reinterpret_cast(arg0_ptr->ob_type)->tp_name)); + py::object numpy_value = + py::object(py::handle(PyTuple_GET_ITEM(args, 0)), true); + paddle::platform::Place place = + CastPyArg2Place(PyTuple_GET_ITEM(args, 1), 1); + bool persistable = CastPyArg2AttrBoolean(PyTuple_GET_ITEM(args, 2), 2); + bool zero_copy = CastPyArg2AttrBoolean(PyTuple_GET_ITEM(args, 3), 3); + std::string act_name = ""; + PyObject* name_obj = PyTuple_GET_ITEM(args, 4); + if (name_obj == Py_None) { + act_name = egr::Controller::Instance().GenerateUniqueName( + "generated_tensor"); + } else { + act_name = CastPyArg2AttrString(PyTuple_GET_ITEM(args, 4), 4); + } + EmptyEagerTensorInitializer(py_tensor_ptr, act_name, place, + persistable); + InitEagerTensorWithNumpyValue(py_tensor_ptr, numpy_value, zero_copy); + return 0; + } else { + PADDLE_THROW(platform::errors::InvalidArgument( + "We only support construct tensor from numpy value or dtype with " + "python args by this initializer, " + "please check your input first and make sure you are on the right " + "way.")); + } + return 0; + } + case (Py_ssize_t)6: { + // case 3 + VLOG(6) << "Calling case3's initializer."; + PyObject* arg0_ptr = PyTuple_GET_ITEM(args, 0); + PADDLE_ENFORCE_EQ( + pybind11::detail::npy_api::get().PyArray_Check_(arg0_ptr), true, + paddle::platform::errors::Fatal( + "We expected initial parametes list like: \n **value: ndarray, " + "\n ** place: paddle::platform::Place, \n ** persistable: bool, " + "\n ** zero_copy: bool, \n ** name: std::string, \n ** " + "stop_gradient: bool. But got value with wrong type: %s", + reinterpret_cast(arg0_ptr->ob_type)->tp_name)); + py::object numpy_value = + py::object(py::handle(PyTuple_GET_ITEM(args, 0)), true); + paddle::platform::Place place = + CastPyArg2Place(PyTuple_GET_ITEM(args, 1), 1); + bool persistable = CastPyArg2AttrBoolean(PyTuple_GET_ITEM(args, 2), 2); + bool zero_copy = CastPyArg2AttrBoolean(PyTuple_GET_ITEM(args, 3), 3); + std::string act_name = ""; + PyObject* name_obj = PyTuple_GET_ITEM(args, 4); + if (name_obj == Py_None) { + act_name = + egr::Controller::Instance().GenerateUniqueName("generated_tensor"); + } else { + act_name = CastPyArg2AttrString(name_obj, 4); + } + bool stop_gradient = CastPyArg2AttrBoolean(PyTuple_GET_ITEM(args, 5), 5); + EmptyEagerTensorInitializer(py_tensor_ptr, act_name, place, persistable, + stop_gradient); + InitEagerTensorWithNumpyValue(py_tensor_ptr, numpy_value, zero_copy); + return 0; + } + default: { + PADDLE_THROW(platform::errors::Fatal( + "Can't not find expected num of args, please check your call, and " + "make sure u call the existed constructor.")); + return 1; + } + } +} + static void eagertensor_dealloc(EagerTensorObject* self) { self->eager_tensor.~EagerTensor(); Py_TYPE(self)->tp_free(reinterpret_cast(self)); @@ -90,7 +491,7 @@ PyTypeObject eager_tensor_type = { 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ - 0, /* tp_init */ + EagerTensorInit, /* tp_init */ 0, /* tp_alloc */ EagerTensorNew, /* tp_new */ 0, /* tp_free */ diff --git a/paddle/fluid/pybind/eager_method.cc b/paddle/fluid/pybind/eager_method.cc index 7305f2f7cfba1cf0e9650b184be27bfb5e7d708b..790969a4b60a6bc73789dbfef6af383bd05121c2 100644 --- a/paddle/fluid/pybind/eager_method.cc +++ b/paddle/fluid/pybind/eager_method.cc @@ -19,6 +19,7 @@ limitations under the License. */ #include "paddle/fluid/eager/api/all.h" #include "paddle/fluid/eager/autograd_meta.h" +#include "paddle/fluid/eager/utils.h" #include "paddle/fluid/memory/allocation/allocator.h" #include "paddle/fluid/memory/memcpy.h" #include "paddle/fluid/platform/enforce.h" @@ -89,19 +90,57 @@ static PyObject* eager_tensor_method_numpy(EagerTensorObject* self, EAGER_CATCH_AND_THROW_RETURN_NULL } -static PyObject* eager_tensor_method_is_initialized(EagerTensorObject* self, - PyObject* args, - PyObject* kwargs) { +static PyObject* eager_tensor_method__is_initialized(EagerTensorObject* self, + PyObject* args, + PyObject* kwargs) { EAGER_SYNC_TRY return ToPyObject(self->eager_tensor.initialized()); EAGER_CATCH_AND_THROW_RETURN_NULL } +static PyObject* eager_tensor_method__copy_to(EagerTensorObject* self, + PyObject* args, + PyObject* kwargs) { + EAGER_SYNC_TRY + bool blocking = CastPyArg2AttrBoolean(PyTuple_GET_ITEM(args, 0), 0); + auto place = CastPyArg2Place(PyTuple_GET_ITEM(args, 1), 1); + auto cp_tensor = + self->eager_tensor.copy_to(pten::TransToPtenBackend(place), blocking); + egr::EagerUtils::autograd_meta(&cp_tensor)->SetStopGradient(true); + egr::EagerUtils::autograd_meta(&cp_tensor) + ->SetPersistable( + egr::EagerUtils::autograd_meta(&(self->eager_tensor))->Persistable()); + return ToPyObject(cp_tensor); + EAGER_CATCH_AND_THROW_RETURN_NULL +} + +static PyObject* eager_tensor_method_copy_(EagerTensorObject* self, + PyObject* args, PyObject* kwargs) { + EAGER_SYNC_TRY + egr::EagerTensor src_tensor = + CastPyArg2EagerTensor(PyTuple_GET_ITEM(args, 0), 0); + bool blocking = CastPyArg2AttrBoolean(PyTuple_GET_ITEM(args, 1), 1); + self->eager_tensor.copy_(src_tensor, blocking); + egr::EagerUtils::autograd_meta(&(self->eager_tensor)) + ->SetStopGradient( + egr::EagerUtils::autograd_meta(&(src_tensor))->StopGradient()); + egr::EagerUtils::autograd_meta(&(self->eager_tensor)) + ->SetPersistable( + egr::EagerUtils::autograd_meta(&(src_tensor))->Persistable()); + Py_INCREF(Py_None); + return Py_None; + EAGER_CATCH_AND_THROW_RETURN_NULL +} + PyMethodDef variable_methods[] = { {"numpy", (PyCFunction)(void (*)(void))eager_tensor_method_numpy, METH_VARARGS | METH_KEYWORDS, NULL}, {"_is_initialized", - (PyCFunction)(void (*)(void))eager_tensor_method_is_initialized, + (PyCFunction)(void (*)(void))eager_tensor_method__is_initialized, + METH_VARARGS | METH_KEYWORDS, NULL}, + {"_copy_to", (PyCFunction)(void (*)(void))eager_tensor_method__copy_to, + METH_VARARGS | METH_KEYWORDS, NULL}, + {"copy_", (PyCFunction)(void (*)(void))eager_tensor_method_copy_, METH_VARARGS | METH_KEYWORDS, NULL}, {NULL, NULL, 0, NULL}}; diff --git a/paddle/fluid/pybind/eager_utils.cc b/paddle/fluid/pybind/eager_utils.cc index c1cd900919252a09add4fae28455d7b8a765f12f..d9da5102262fee15667e952d7fbc586285490e49 100644 --- a/paddle/fluid/pybind/eager_utils.cc +++ b/paddle/fluid/pybind/eager_utils.cc @@ -213,6 +213,47 @@ std::vector CastPyArg2VectorOfEagerTensor(PyObject* obj, item, reinterpret_cast(p_eager_tensor_type))) { result.emplace_back( reinterpret_cast(item)->eager_tensor); + } else { + PADDLE_THROW(platform::errors::InvalidArgument( + "argument (position %d) must be " + "list of EagerTensor, but got %s at pos %d", + arg_pos + 1, + reinterpret_cast(item->ob_type)->tp_name, i)); + } + } + } else { + PADDLE_THROW(platform::errors::InvalidArgument( + "argument (position %d) must be " + "list or tuple, but got %s", + arg_pos + 1, reinterpret_cast(obj->ob_type)->tp_name)); + } + return result; +} + +std::vector CastPyArg2VectorOfInt(PyObject* obj, size_t arg_pos) { + std::vector result; + if (PyList_Check(obj)) { + Py_ssize_t len = PyList_Size(obj); + PyObject* item = nullptr; + for (Py_ssize_t i = 0; i < len; i++) { + item = PyList_GetItem(obj, i); + if (PyObject_CheckLongOrConvertToLong(&item)) { + result.emplace_back(static_cast(PyLong_AsLong(item))); + } else { + PADDLE_THROW(platform::errors::InvalidArgument( + "argument (position %d) must be " + "list of int, but got %s at pos %d", + arg_pos + 1, + reinterpret_cast(item->ob_type)->tp_name, i)); + } + } + } else if (PyTuple_Check(obj)) { + Py_ssize_t len = PyTuple_Size(obj); + PyObject* item = nullptr; + for (Py_ssize_t i = 0; i < len; i++) { + item = PyTuple_GetItem(obj, i); + if (PyObject_CheckLongOrConvertToLong(&item)) { + result.emplace_back(static_cast(PyLong_AsLong(item))); } else { PADDLE_THROW(platform::errors::InvalidArgument( "argument (position %d) must be " diff --git a/paddle/fluid/pybind/eager_utils.h b/paddle/fluid/pybind/eager_utils.h index 1dea81d0263233f2a580cadc59d1690c394733ae..e493e06d7d7df7907253a5d9cba71b9e27cedb84 100644 --- a/paddle/fluid/pybind/eager_utils.h +++ b/paddle/fluid/pybind/eager_utils.h @@ -35,6 +35,7 @@ egr::EagerTensor CastPyArg2EagerTensor(PyObject* obj, ssize_t arg_pos); std::vector CastPyArg2VectorOfEagerTensor(PyObject* obj, ssize_t arg_pos); platform::Place CastPyArg2Place(PyObject* obj, ssize_t arg_pos); +std::vector CastPyArg2VectorOfInt(PyObject* obj, size_t arg_pos); framework::proto::VarType::Type CastPyArg2ProtoType(PyObject* obj, ssize_t arg_pos); PyObject* ToPyObject(int value); diff --git a/paddle/pten/api/include/tensor.h b/paddle/pten/api/include/tensor.h index 3471a87b21cadb612226ac583913b0ab604a1db0..c8ef22c2ecda2b381625cc2868301b8de6ea2cfd 100644 --- a/paddle/pten/api/include/tensor.h +++ b/paddle/pten/api/include/tensor.h @@ -341,7 +341,9 @@ class PADDLE_API Tensor final { void set_name(const std::string& name) { name_ = name; } /* Part 5: Data Transform methods */ - + /* Alert!!!!: All copy method can only deep copy impl, autograd info only be + * copied */ + /* out of pten */ /** * @brief Copy the current Tensor data to the specified device * and return the new Tensor. It's usually used to set the input tensor data. @@ -361,12 +363,20 @@ class PADDLE_API Tensor final { /** * @brief Transfer the current Tensor to the specified device and return. * - * @param place, the target place of which the tensor will copy to. + * @param backend, The target backend of which the tensor will copy to. + * @param blocking, Should we copy this in sync way. * @return Tensor */ - // TODO(chenweihang): replace Backend by new Place Tensor copy_to(Backend backend, bool blocking) const; + /** + * @brief Transfer the source Tensor to current Tensor. + * + * @param src, the source Tensor to be copied. + * @param blocking, Should we copy this in sync way. + * @return void + */ + void copy_(const Tensor& src, const bool blocking); /** * @brief Cast datatype from one to another * diff --git a/paddle/pten/api/lib/tensor.cc b/paddle/pten/api/lib/tensor.cc index f6cccf0b357ce16d6a2672c87ee4e031b29c839a..6ecc46ca8b53f140d1e8b286f9f56e6596ca4412 100644 --- a/paddle/pten/api/lib/tensor.cc +++ b/paddle/pten/api/lib/tensor.cc @@ -24,6 +24,7 @@ limitations under the License. */ #include "paddle/pten/api/lib/utils/allocator.h" #include "paddle/pten/api/lib/utils/storage.h" #include "paddle/pten/core/compat_utils.h" +#include "paddle/pten/core/convert_utils.h" #include "paddle/pten/core/dense_tensor.h" #include "paddle/pten/core/tensor_base.h" #include "paddle/pten/core/tensor_meta.h" @@ -321,6 +322,31 @@ Tensor Tensor::copy_to(Backend backend, bool blocking) const { return experimental::copy_to(*this, backend, blocking); } +void Tensor::copy_(const Tensor &src, bool blocking) { + if (!src.is_initialized()) { + return; + } + VLOG(3) << "Deep copy Tensor from " << src.name() << " to " << name(); + if (defined()) { + PADDLE_ENFORCE_EQ(dtype(), + src.dtype(), + platform::errors::PreconditionNotMet( + "Tensor %s has different data type with Tensor %s, " + "Tensor Copy cannot be performed!", + name(), + src.name())); + PADDLE_ENFORCE_EQ(impl()->type_info().id(), + src.impl()->type_info().id(), + platform::errors::PreconditionNotMet( + "Tensor %s has different type with Tensor %s, Tensor " + "Copy cannot be performed!", + name(), + src.name())); + } + auto copy_tensor = + src.copy_to(pten::TransToPtenBackend(src.inner_place()), blocking); + set_impl(copy_tensor.impl()); +} Tensor Tensor::cast(DataType target_type) const { return experimental::cast(*this, target_type); } diff --git a/paddle/pten/api/lib/utils/tensor_utils.cc b/paddle/pten/api/lib/utils/tensor_utils.cc index f2b6e4841aab24e9344de644a00c003ec395fb61..b248cd209899b330088cedff380980334a7fbb28 100644 --- a/paddle/pten/api/lib/utils/tensor_utils.cc +++ b/paddle/pten/api/lib/utils/tensor_utils.cc @@ -389,6 +389,28 @@ void ReMakePtenDenseTensor(const paddle::framework::Tensor& src, } } +void ReMakePtenDenseTensor(const paddle::framework::LoDTensor& src, + pten::DenseTensor* dst) { + auto* meta = pten::CompatibleDenseTensorUtils::GetMutableMeta(dst); + meta->dims = src.dims(); + // Since the type of DenseTensorMeta is const, const_cast must be used + const_cast(meta->dtype) = pten::TransToPtenDataType(src.type()); + // Since the type of DenseTensorMeta is const, const_cast must be used + const_cast(meta->layout) = + pten::TransToPtenDataLayout(src.layout()); + + auto* shared_storage = static_cast( + pten::CompatibleDenseTensorUtils::UnsafeGetMutableStorage(dst)); + PADDLE_ENFORCE_NOT_NULL( + shared_storage, + platform::errors::NotFound( + "Target DenseTensor's shared storage is nullptr.")); + + if (src.IsInitialized()) { + shared_storage->ResetAllocation(src.Holder(), src.offset()); + } +} + void ReMakePtenDenseTensor(const paddle::framework::LoDTensor& src, const pten::TensorArgDef& arg_def, pten::DenseTensor* dst) { diff --git a/paddle/pten/api/lib/utils/tensor_utils.h b/paddle/pten/api/lib/utils/tensor_utils.h index 6397ca369ce75514f583a1d808c7d01d5b512b62..32b7c377ebfdef1f7b7733ae2a32f70f2a33721a 100644 --- a/paddle/pten/api/lib/utils/tensor_utils.h +++ b/paddle/pten/api/lib/utils/tensor_utils.h @@ -72,6 +72,8 @@ void MovesSharedStorage(pten::DenseTensor* src, * the overhead caused by frequent construction and destruction of the * DenseTensor. */ +void ReMakePtenDenseTensor(const paddle::framework::LoDTensor& src, + pten::DenseTensor* dst); void ReMakePtenDenseTensor(const paddle::framework::Tensor& src, const pten::TensorArgDef& arg_def, diff --git a/python/paddle/fluid/tests/unittests/test_egr_python_api.py b/python/paddle/fluid/tests/unittests/test_egr_python_api.py index 8d4a839df6cf4b1642f4e772151e2bd54a0dcaad..08a68ca246f4061e9c7312daed4250fbc82453da 100644 --- a/python/paddle/fluid/tests/unittests/test_egr_python_api.py +++ b/python/paddle/fluid/tests/unittests/test_egr_python_api.py @@ -81,6 +81,173 @@ class EagerDtypeTestCase(unittest.TestCase): class EagerTensorPropertiesTestCase(unittest.TestCase): + def constructor(self, place): + egr_tensor = core.eager.EagerTensor() + self.assertEqual(egr_tensor.persistable, False) + self.assertTrue("generated" in egr_tensor.name) + self.assertEqual(egr_tensor.shape, []) + self.assertEqual(egr_tensor.dtype, core.VarDesc.VarType.FP32) + self.assertEqual(egr_tensor.stop_gradient, True) + + egr_tensor0 = core.eager.EagerTensor( + core.VarDesc.VarType.FP32, [4, 16, 16, 32], "test_eager_tensor", + core.VarDesc.VarType.LOD_TENSOR, True) + self.assertEqual(egr_tensor0.persistable, True) + self.assertEqual(egr_tensor0.name, "test_eager_tensor") + self.assertEqual(egr_tensor0.shape, [4, 16, 16, 32]) + self.assertEqual(egr_tensor0.dtype, core.VarDesc.VarType.FP32) + + arr0 = np.random.rand(4, 16, 16, 32).astype('float32') + egr_tensor1 = core.eager.EagerTensor(arr0, place, True, False, + "numpy_tensor1", False) + self.assertEqual(egr_tensor1.persistable, True) + self.assertEqual(egr_tensor1.name, "numpy_tensor1") + self.assertEqual(egr_tensor1.shape, [4, 16, 16, 32]) + self.assertEqual(egr_tensor1.dtype, core.VarDesc.VarType.FP32) + self.assertEqual(egr_tensor1.stop_gradient, False) + self.assertTrue(egr_tensor1.place._equals(place)) + self.assertTrue(np.array_equal(egr_tensor1.numpy(), arr0)) + + arr1 = np.random.randint(100, size=(4, 16, 16, 32), dtype=np.int64) + egr_tensor2 = core.eager.EagerTensor(arr1, place, False, True, + "numpy_tensor2", True) + self.assertEqual(egr_tensor2.persistable, False) + self.assertEqual(egr_tensor2.name, "numpy_tensor2") + self.assertEqual(egr_tensor2.shape, [4, 16, 16, 32]) + self.assertEqual(egr_tensor2.dtype, core.VarDesc.VarType.INT64) + self.assertEqual(egr_tensor2.stop_gradient, True) + self.assertTrue(egr_tensor2.place._equals(place)) + self.assertTrue(np.array_equal(egr_tensor2.numpy(), arr1)) + + arr2 = np.random.rand(4, 16, 16, 32, 64).astype('float32') + egr_tensor3 = core.eager.EagerTensor(arr2) + self.assertEqual(egr_tensor3.persistable, False) + self.assertTrue("generated_tensor" in egr_tensor3.name) + self.assertEqual(egr_tensor3.shape, [4, 16, 16, 32, 64]) + self.assertEqual(egr_tensor3.dtype, core.VarDesc.VarType.FP32) + self.assertEqual(egr_tensor3.stop_gradient, True) + self.assertTrue( + egr_tensor3.place._equals( + paddle.fluid.framework._current_expected_place())) + self.assertTrue(np.array_equal(egr_tensor3.numpy(), arr2)) + + egr_tensor3.stop_gradient = False + egr_tensor4 = core.eager.EagerTensor(egr_tensor3) + self.assertEqual(egr_tensor4.persistable, False) + self.assertTrue("generated_tensor" in egr_tensor4.name) + self.assertEqual(egr_tensor4.shape, egr_tensor3.shape) + self.assertEqual(egr_tensor4.dtype, egr_tensor3.dtype) + self.assertEqual(egr_tensor4.stop_gradient, True) + self.assertTrue( + egr_tensor4.place._equals( + paddle.fluid.framework._current_expected_place())) + self.assertTrue( + np.array_equal(egr_tensor4.numpy(), egr_tensor3.numpy())) + + arr4 = np.random.rand(4, 16, 16, 32).astype('float32') + egr_tensor5 = core.eager.EagerTensor(arr4, place) + self.assertEqual(egr_tensor5.persistable, False) + self.assertTrue("generated_tensor" in egr_tensor5.name) + self.assertEqual(egr_tensor5.shape, [4, 16, 16, 32]) + self.assertEqual(egr_tensor5.dtype, core.VarDesc.VarType.FP32) + self.assertEqual(egr_tensor5.stop_gradient, True) + self.assertTrue(egr_tensor5.place._equals(place)) + self.assertTrue(np.array_equal(egr_tensor5.numpy(), arr4)) + + egr_tensor6 = core.eager.EagerTensor(egr_tensor5, core.CPUPlace()) + self.assertEqual(egr_tensor6.persistable, False) + self.assertTrue("generated_tensor" in egr_tensor6.name) + self.assertEqual(egr_tensor6.shape, [4, 16, 16, 32]) + self.assertEqual(egr_tensor6.dtype, core.VarDesc.VarType.FP32) + self.assertEqual(egr_tensor6.stop_gradient, True) + self.assertEqual(egr_tensor6.place.is_cpu_place(), True) + self.assertTrue( + np.array_equal(egr_tensor6.numpy(), egr_tensor5.numpy())) + + egr_tensor7 = core.eager.EagerTensor(arr4, place, True) + self.assertEqual(egr_tensor7.persistable, True) + self.assertTrue("generated_tensor" in egr_tensor7.name) + self.assertEqual(egr_tensor7.shape, [4, 16, 16, 32]) + self.assertEqual(egr_tensor7.dtype, core.VarDesc.VarType.FP32) + self.assertEqual(egr_tensor7.stop_gradient, True) + self.assertTrue(egr_tensor7.place._equals(place)) + self.assertTrue(np.array_equal(egr_tensor7.numpy(), arr4)) + + egr_tensor8 = core.eager.EagerTensor(egr_tensor6, place, "egr_tensor8") + self.assertEqual(egr_tensor8.persistable, False) + self.assertEqual(egr_tensor8.name, "egr_tensor8") + self.assertEqual(egr_tensor8.shape, [4, 16, 16, 32]) + self.assertEqual(egr_tensor8.dtype, core.VarDesc.VarType.FP32) + self.assertEqual(egr_tensor8.stop_gradient, True) + self.assertTrue(egr_tensor8.place._equals(place)) + self.assertTrue( + np.array_equal(egr_tensor8.numpy(), egr_tensor5.numpy())) + + egr_tensor9 = core.eager.EagerTensor(arr4, place, True, True) + self.assertEqual(egr_tensor9.persistable, True) + self.assertTrue("generated_tensor" in egr_tensor9.name) + self.assertEqual(egr_tensor9.shape, [4, 16, 16, 32]) + self.assertEqual(egr_tensor9.dtype, core.VarDesc.VarType.FP32) + self.assertEqual(egr_tensor9.stop_gradient, True) + self.assertTrue(egr_tensor9.place._equals(place)) + self.assertTrue(np.array_equal(egr_tensor9.numpy(), arr4)) + + def test_constructor(self): + print("Test_constructor") + paddle.set_device("cpu") + place_list = [core.CPUPlace()] + if core.is_compiled_with_cuda(): + place_list.append(core.CUDAPlace(0)) + with _test_eager_guard(): + for p in place_list: + self.constructor(p) + + def test_copy_and_copy_to(self): + print("Test_copy_and_copy_to") + with _test_eager_guard(): + paddle.set_device("cpu") + arr = np.ones([4, 16, 16, 32]).astype('float32') + arr1 = np.zeros([4, 16]).astype('float32') + arr2 = np.ones([4, 16, 16, 32]).astype('float32') + np.ones( + [4, 16, 16, 32]).astype('float32') + tensor = paddle.to_tensor(arr, core.VarDesc.VarType.FP32, + core.CPUPlace()) + self.assertEqual(tensor.stop_gradient, True) + tensor.stop_gradient = False + print("Set persistable") + tensor.persistable = False + tensor1 = paddle.to_tensor(arr1, core.VarDesc.VarType.FP32, + core.CPUPlace()) + tensor1.persistable = True + self.assertEqual(tensor1.stop_gradient, True) + self.assertTrue(np.array_equal(tensor.numpy(), arr)) + print("Test copy_") + tensor.copy_(tensor1, True) + self.assertEqual(tensor.persistable, True) + self.assertEqual(tensor.shape, [4, 16]) + self.assertEqual(tensor.dtype, core.VarDesc.VarType.FP32) + self.assertTrue(np.array_equal(tensor.numpy(), arr1)) + + print("Test _copy_to") + tensor2 = paddle.to_tensor(arr2, core.VarDesc.VarType.FP32, + core.CPUPlace()) + self.assertTrue(np.array_equal(tensor2.numpy(), arr2)) + self.assertTrue(tensor2.place.is_cpu_place()) + tensor2.persistable = True + tensor2.stop_gradient = False + if core.is_compiled_with_cuda(): + tensor3 = tensor2._copy_to(True, core.CUDAPlace(0)) + self.assertTrue(np.array_equal(tensor3.numpy(), arr2)) + self.assertTrue(tensor3.persistable, True) + self.assertTrue(tensor3.stop_gradient, True) + self.assertTrue(tensor3.place.is_gpu_place()) + else: + tensor3 = tensor2._copy_to(True, core.CPUPlace()) + self.assertTrue(np.array_equal(tensor3.numpy(), arr2)) + self.assertTrue(tensor3.persistable, True) + self.assertTrue(tensor3.stop_gradient, True) + self.assertTrue(tensor3.place.is_cpu_place()) + def test_properties(self): print("Test_properties") with _test_eager_guard():