未验证 提交 d3686471 编写于 作者: W wanghuancoder 提交者: GitHub

[Eager] fix some eager logic (#38576)

* Rearranged Eager AutoCodeGen directory structure

* Removed USE_OP in Eager AutoCodeGen

* Enabled generation for Operators without Grad/Inputs/Outputs

* Resolved operators without input

* Fixed merge conflicts

* Enabled Eager AutoCodeGen for 10+ more operators

* Refactored Eager AutoCodeGen with more organized helper objects

* Enabled Eager AutoCodeGen for operators with multiple OpBases

* Adjusted Eager AutoCodeGen to Enable Passing Output Tensor as Input Argument

* Handled Dispensable Inputs/Outputs in Eager AutoCodeGen

* Adjusted function generation/call between Python-C API & Dygraph API

* Synchronized auto-generated Python-C API with Dygraph Forward Functions

* support more eager tensor api

* fix merge compile error

* fix compile error and fit develop code

* support pure CPU

* fix some logic error in eager_mode

* support _varbase_creator in eager mode

* Added safe_initialized interface to EagerTensor for use in processing dispensable inputs

* for eager mode

* refine

* support multiple constructor for eager tensor

* add place related code

* polish code

* specific randint with dtype of int64

* Support pure cpu test

* eager logic

* refine test in pure cpu

* eager logic

* eager logic

* eager logic, test=develop

* skip core.eager when in inference, test=develop

* refine, test=develop

* refine, test=develop

* call RetainGrad after run forward kernel, test=develop

* refine, test=develop

* support dygraph util, meta, guard test

* eager test case

* support inference test

* refine test and fix initializer failed

* modify eagertensor patch method

* add eagertensor.clear_grandint, test=develop

* refine, test=develop

* refine, test=develop

* refine, test=develop

* support create varbase and fix retain grad error

* call monkey_patch_varbase in _test_eager_guard, test=develop

* fix windows error

* split clear_gradient to clear_gradient and zero_grads, test=develop

* refine, test=develop

* refine, test=develop

* support test_imperative_basic test in eager mode

* remove additional log in variable.h

* remove additional log in variable.h

* remove additional code create in merge

* eager

* fix some eager logic, test=develop

* refine, test=develop

* refine, test=develop

* refine, test=develop

* refine, test=develop

* refine, test=develop
Co-authored-by: Njim19930609 <jim19930609@gmail.com>
Co-authored-by: NJiabinYang <360788950@qq.com>
上级 ffbc2122
......@@ -28,6 +28,9 @@
static void CopyOrAddTensor(egr::EagerTensor* tensor,
const egr::EagerTensor& t) {
if (t.Var().IsInitialized()) {
const_cast<egr::EagerTensor*>(&t)->SyncToTensor();
}
if (!tensor->defined() || !tensor->initialized()) {
// Simply copy tensor->impl
*tensor = t;
......
......@@ -32,7 +32,7 @@ class GradNodeAccumulation : public GradNodeBase {
void RetainGrad(
const std::function<egr::EagerTensor(const egr::EagerTensor&)>& hook);
egr::EagerTensor Grad() { return accumulated_grad; }
egr::EagerTensor* Grad() { return &accumulated_grad; }
private:
egr::EagerTensor accumulated_grad;
......
......@@ -239,8 +239,8 @@ class EagerTensor final {
auto tensor_dense =
std::dynamic_pointer_cast<pten::DenseTensor>(tensor_->impl());
if (tensor_dense) {
paddle::experimental::MovesStorage(tensor_dense.get(),
framework_tensor);
paddle::experimental::SharesStorage(tensor_dense.get(),
framework_tensor);
} else {
PADDLE_THROW(paddle::platform::errors::Fatal(
"Unrecognized egr::EagerTensor type, only "
......@@ -258,27 +258,23 @@ class EagerTensor final {
/** Part 11: Sync paddle::framework::Variable with pten::Tensor **/
void SyncToTensor() {
// Synchronize allocation only once.
if (!this->defined() || !this->initialized()) {
// TODO(jiabin): Support selected rows later.
if (var_.IsInitialized()) {
if (var_.IsType<paddle::framework::LoDTensor>()) {
SetImplWithLegacyTensor<paddle::framework::LoDTensor,
pten::DenseTensor>();
} else if (var_.IsType<paddle::framework::Tensor>()) {
SetImplWithLegacyTensor<paddle::framework::Tensor,
pten::DenseTensor>();
} else {
PADDLE_THROW(paddle::platform::errors::Fatal(
"Unable to fetch underlying tensor "
"from VarBase, only LoDTensor and "
"Tensor are supported for now"));
}
if (var_.IsInitialized()) {
if (var_.IsType<paddle::framework::LoDTensor>()) {
SetImplWithLegacyTensor<paddle::framework::LoDTensor,
pten::DenseTensor>();
} else if (var_.IsType<paddle::framework::Tensor>()) {
SetImplWithLegacyTensor<paddle::framework::Tensor, pten::DenseTensor>();
} else {
PADDLE_THROW(paddle::platform::errors::Fatal(
"Can not Sync EagerTensor %s whose paddle::framework::Variable is "
"not initialized!",
name()));
PADDLE_THROW(
paddle::platform::errors::Fatal("Unable to fetch underlying tensor "
"from VarBase, only LoDTensor and "
"Tensor are supported for now"));
}
} else {
PADDLE_THROW(paddle::platform::errors::Fatal(
"Can not Sync EagerTensor %s whose paddle::framework::Variable is "
"not initialized!",
name()));
}
}
......@@ -296,8 +292,16 @@ class EagerTensor final {
template <typename LEGACY_TYPE, typename TYPE>
void SetImplWithLegacyTensor() {
const auto& framework_tensor = var_.Get<LEGACY_TYPE>();
this->set_impl(
std::move(paddle::experimental::MakePtenDenseTensor(framework_tensor)));
if (this->initialized()) {
VLOG(8) << "Sync Var to initialized tensor for: " << name();
paddle::experimental::ReMakePtenDenseTensor(
framework_tensor,
static_cast<pten::DenseTensor*>(this->impl().get()));
} else {
VLOG(8) << "Sync Var to uninitialized tensor for: " << name();
this->set_impl(std::move(
paddle::experimental::MakePtenDenseTensor(framework_tensor)));
}
var_.Clear();
}
......
......@@ -118,7 +118,7 @@ TEST(EagerTensor, MemberFunction) {
CHECK_EQ(et3.Var().Get<paddle::framework::LoDTensor>().data<float>()[1],
10.0f);
VLOG(6) << "SyncToTensor";
CHECK(et3.initialized() == false);
CHECK(et3.initialized() == true);
et3.SyncToTensor();
CHECK(et3.initialized() == true);
VLOG(6) << "Check Tensor";
......
......@@ -87,8 +87,8 @@ void benchmark_eager_intermediate_matmul(const EagerTensor& X,
// Examine Forward Grad (w.r.t max_num_runs = 2)
eager_test::CompareVariableWithValue<float>(input_tensor0, 16);
// Examine Backward Grad (w.r.t max_num_runs = 2)
eager_test::CompareGradVariableWithValue<float>(X, 16);
eager_test::CompareGradVariableWithValue<float>(Y, 16);
eager_test::CompareGradTensorWithValue<float>(X, 16);
eager_test::CompareGradTensorWithValue<float>(Y, 16);
}
}
......@@ -121,8 +121,8 @@ void benchmark_eager_intermediate_mlp(const EagerTensor& X,
eager_test::CompareVariableWithValue<float>(Out, result["Out"]);
// Examine Backward Grad (w.r.t max_num_runs = 2)
eager_test::CompareGradVariableWithValue<float>(X, result["GradX"]);
eager_test::CompareGradVariableWithValue<float>(Ws[0], result["GradW"]);
eager_test::CompareGradTensorWithValue<float>(X, result["GradX"]);
eager_test::CompareGradTensorWithValue<float>(Ws[0], result["GradW"]);
}
}
......
......@@ -54,7 +54,7 @@ TEST(Generated, Sigmoid) {
RunBackward(target_tensors, {});
VLOG(6) << "Finish Backward";
eager_test::CompareGradVariableWithValue<float>(tensor, 0.25);
eager_test::CompareGradTensorWithValue<float>(tensor, 0.25);
}
TEST(Generated, Matmul_v2) {
......@@ -85,8 +85,8 @@ TEST(Generated, Matmul_v2) {
std::vector<egr::EagerTensor> target_tensors = {output_tensor};
RunBackward(target_tensors, {});
eager_test::CompareGradVariableWithValue<float>(X, 2.0 * 20);
eager_test::CompareGradVariableWithValue<float>(Y, 3.0 * 4);
eager_test::CompareGradTensorWithValue<float>(X, 2.0 * 20);
eager_test::CompareGradTensorWithValue<float>(Y, 3.0 * 4);
}
TEST(Generated, ElementwiseAdd) {
......@@ -116,8 +116,8 @@ TEST(Generated, ElementwiseAdd) {
std::vector<egr::EagerTensor> target_tensors = {output_tensor};
RunBackward(target_tensors, {});
eager_test::CompareGradVariableWithValue<float>(X, 1.0);
eager_test::CompareGradVariableWithValue<float>(Y, 1.0);
eager_test::CompareGradTensorWithValue<float>(X, 1.0);
eager_test::CompareGradTensorWithValue<float>(Y, 1.0);
}
} // namespace egr
......
......@@ -35,7 +35,7 @@ limitations under the License. */
namespace paddle {
namespace pybind {
extern PyTypeObject* pEagerTensorType;
extern PyTypeObject* p_eager_tensor_type;
static PyObject* eager_tensor_method_numpy(EagerTensorObject* self,
PyObject* args, PyObject* kwargs) {
......@@ -167,7 +167,7 @@ static PyObject* eager_tensor__clear_gradient(EagerTensorObject* self,
EAGER_SYNC_TRY
VLOG(4) << "ClearGradient " << self->eager_tensor.name();
egr::EagerTensor grad;
egr::EagerTensor* grad;
if (egr::egr_utils_api::IsLeafTensor(self->eager_tensor)) {
// Add RetainGrad as PostHook to AccumulationNode
std::shared_ptr<egr::GradNodeBase> grad_node =
......@@ -182,14 +182,14 @@ static PyObject* eager_tensor__clear_gradient(EagerTensorObject* self,
grad = accumulation_grad_node->Grad();
} else {
auto meta = egr::EagerUtils::unsafe_autograd_meta(self->eager_tensor);
grad = meta->Grad();
grad = meta->MutableGrad();
}
if (grad.initialized()) {
if (grad->initialized()) {
VLOG(4) << "Gradient of " << self->eager_tensor.name()
<< " is initialized, will be released.";
auto dense_tensor =
std::dynamic_pointer_cast<pten::DenseTensor>(grad.impl());
std::dynamic_pointer_cast<pten::DenseTensor>(grad->impl());
dense_tensor->release();
}
Py_INCREF(Py_None);
......@@ -202,7 +202,6 @@ static PyObject* eager_tensor__zero_grads(EagerTensorObject* self,
EAGER_TRY
VLOG(4) << "ZeroGrads " << self->eager_tensor.name();
egr::EagerTensor grad;
if (egr::egr_utils_api::IsLeafTensor(self->eager_tensor)) {
// Add RetainGrad as PostHook to AccumulationNode
std::shared_ptr<egr::GradNodeBase> grad_node =
......@@ -214,21 +213,54 @@ static PyObject* eager_tensor__zero_grads(EagerTensorObject* self,
"with type: GradNodeAccumulation"));
auto accumulation_grad_node =
std::dynamic_pointer_cast<egr::GradNodeAccumulation>(grad_node);
grad = accumulation_grad_node->Grad();
if (accumulation_grad_node->Grad()->initialized()) {
accumulation_grad_node->Grad()->set_tensor(
std::make_shared<paddle::experimental::Tensor>(
paddle::experimental::zeros_like(
*(accumulation_grad_node->Grad()->Tensor().get()))));
}
} else {
auto meta = egr::EagerUtils::unsafe_autograd_meta(self->eager_tensor);
grad = meta->Grad();
if (meta->MutableGrad()->initialized()) {
meta->MutableGrad()->set_tensor(
std::make_shared<paddle::experimental::Tensor>(
paddle::experimental::zeros_like(
*(meta->MutableGrad()->Tensor().get()))));
}
}
if (grad.initialized()) {
grad.set_tensor(std::make_shared<paddle::experimental::Tensor>(
paddle::experimental::zeros_like(*(grad.Tensor().get()))));
}
Py_INCREF(Py_None);
return Py_None;
EAGER_CATCH_AND_THROW_RETURN_NULL
}
static PyObject* eager_tensor_method_detach(EagerTensorObject* self,
PyObject* args, PyObject* kwargs) {
EAGER_SYNC_TRY
PADDLE_ENFORCE_EQ(
self->eager_tensor.initialized(), true,
platform::errors::InvalidArgument("Tensor %s has not been initialized!",
self->eager_tensor.name()));
PyObject* obj = p_eager_tensor_type->tp_alloc(p_eager_tensor_type, 0);
if (obj) {
auto v = reinterpret_cast<EagerTensorObject*>(obj);
new (&(v->eager_tensor)) egr::EagerTensor();
v->eager_tensor.set_impl(self->eager_tensor.impl());
v->eager_tensor.set_name(egr::Controller::Instance().GenerateUniqueName());
auto autograd_meta_src =
egr::EagerUtils::autograd_meta(&(self->eager_tensor));
auto autograd_meta = egr::EagerUtils::autograd_meta(&(v->eager_tensor));
autograd_meta->SetPersistable(autograd_meta_src->Persistable());
} else {
PADDLE_THROW(platform::errors::Fatal(
"tp_alloc return null, can not new a PyObject."));
}
return obj;
EAGER_CATCH_AND_THROW_RETURN_NULL
}
PyMethodDef variable_methods[] = {
{"numpy", (PyCFunction)(void (*)(void))eager_tensor_method_numpy,
METH_VARARGS | METH_KEYWORDS, NULL},
......@@ -246,6 +278,8 @@ PyMethodDef variable_methods[] = {
METH_VARARGS | METH_KEYWORDS, NULL},
{"_zero_grads", (PyCFunction)(void (*)(void))eager_tensor__zero_grads,
METH_VARARGS | METH_KEYWORDS, NULL},
{"detach", (PyCFunction)(void (*)(void))eager_tensor_method_detach,
METH_VARARGS | METH_KEYWORDS, NULL},
{NULL, NULL, 0, NULL}};
} // namespace pybind
......
......@@ -63,7 +63,6 @@ PyObject* eager_tensor_properties_get_grad(EagerTensorObject* self,
void* closure) {
EAGER_SYNC_TRY
if (egr::egr_utils_api::IsLeafTensor(self->eager_tensor)) {
// Add RetainGrad as PostHook to AccumulationNode
std::shared_ptr<egr::GradNodeBase> grad_node =
egr::EagerUtils::grad_node(self->eager_tensor);
PADDLE_ENFORCE(
......@@ -73,7 +72,7 @@ PyObject* eager_tensor_properties_get_grad(EagerTensorObject* self,
"with type: GradNodeAccumulation"));
auto accumulation_grad_node =
std::dynamic_pointer_cast<egr::GradNodeAccumulation>(grad_node);
return ToPyObject(accumulation_grad_node->Grad());
return ToPyObject(*accumulation_grad_node->Grad());
} else {
VLOG(6) << "Get grad for tensor: " << self->eager_tensor.name();
auto meta = egr::EagerUtils::unsafe_autograd_meta(self->eager_tensor);
......@@ -82,6 +81,27 @@ PyObject* eager_tensor_properties_get_grad(EagerTensorObject* self,
EAGER_CATCH_AND_THROW_RETURN_NULL
}
int eager_tensor_properties_set_grad(EagerTensorObject* self, PyObject* value,
void* closure) {
EAGER_SYNC_TRY
auto src = CastPyArg2EagerTensor(value, 0);
PADDLE_ENFORCE(
egr::egr_utils_api::IsLeafTensor(self->eager_tensor),
paddle::platform::errors::Fatal("Only leaf Tensor can be set grad."));
std::shared_ptr<egr::GradNodeBase> grad_node =
egr::EagerUtils::grad_node(self->eager_tensor);
PADDLE_ENFORCE(
grad_node.get() != nullptr,
paddle::platform::errors::Fatal("Detected NULL grad_node"
"Leaf tensor should have had grad_node "
"with type: GradNodeAccumulation"));
auto accumulation_grad_node =
std::dynamic_pointer_cast<egr::GradNodeAccumulation>(grad_node);
accumulation_grad_node->Grad()->copy_(src, true);
return 0;
EAGER_CATCH_AND_THROW_RETURN_ZERO
}
int eager_tensor_properties_set_stop_gradient(EagerTensorObject* self,
PyObject* value, void* closure) {
EAGER_SYNC_TRY
......@@ -147,8 +167,8 @@ PyObject* eager_tensor_properties_get_dtype(EagerTensorObject* self,
}
struct PyGetSetDef variable_properties[] = {
{"grad", (getter)eager_tensor_properties_get_grad, nullptr, nullptr,
nullptr},
{"grad", (getter)eager_tensor_properties_get_grad,
(setter)eager_tensor_properties_set_grad, nullptr, nullptr},
{"name", (getter)eager_tensor_properties_get_name,
(setter)eager_tensor_properties_set_name, nullptr, nullptr},
{"stop_gradient", (getter)eager_tensor_properties_get_stop_gradient,
......
......@@ -22,7 +22,7 @@ import paddle
from .. import framework
from .. import core
from .. import unique_name
from ..framework import Variable, Parameter, ParamBase, _getitem_impl_, _setitem_impl_, _in_eager_mode
from ..framework import Variable, Parameter, ParamBase, _getitem_impl_, _setitem_impl_, _in_eager_mode, EagerParamBase
from .base import switch_to_static_graph
from .math_op_patch import monkey_patch_math_varbase
from .parallel import scale_loss
......@@ -149,7 +149,7 @@ def monkey_patch_varbase():
out = linear(t) # call with different weight
"""
if _in_eager_mode():
if core._in_eager_mode():
base_tensor = core.eager.EagerTensor
else:
base_tensor = core.VarBase
......@@ -238,7 +238,7 @@ def monkey_patch_varbase():
"""
if framework.in_dygraph_mode():
if grad_tensor is not None:
if _in_eager_mode():
if core._in_eager_mode():
assert isinstance(
grad_tensor, core.eager.EagerTensor
), "The type of grad_tensor must be paddle.Tensor"
......@@ -250,7 +250,7 @@ def monkey_patch_varbase():
"Tensor shape not match, Tensor of grad_tensor [ {} ] with shape {} mismatch Tensor [ {} ] with shape {}".format(
grad_tensor.name, grad_tensor.shape, self.name, self.shape)
if _in_eager_mode():
if core._in_eager_mode():
if grad_tensor is None:
grad_tensor = []
else:
......@@ -258,7 +258,7 @@ def monkey_patch_varbase():
if paddle.is_compiled_with_xpu() or paddle.is_compiled_with_npu():
# TODO(liuyuhui): Currently only for xpu. Will be removed in the future.
scaled_loss = scale_loss(self)
if _in_eager_mode():
if core._in_eager_mode():
core.eager.run_backward([scaled_loss], grad_tensor,
retain_graph)
else:
......@@ -266,7 +266,7 @@ def monkey_patch_varbase():
retain_graph,
framework._dygraph_tracer())
else:
if _in_eager_mode():
if core._in_eager_mode():
core.eager.run_backward([self], grad_tensor, retain_graph)
else:
core.dygraph_run_backward([self], [grad_tensor],
......@@ -305,7 +305,7 @@ def monkey_patch_varbase():
# [500.]
"""
if _in_eager_mode():
if core._in_eager_mode():
if not self.grad._is_initialized():
return None
# TODO(wanghuancoder) support SELECTED_ROWS
......@@ -587,7 +587,7 @@ def monkey_patch_varbase():
# [[0.30574632, 0.55739117, 0.30902600, 0.39413780, 0.44830436],
# [0.79010487, 0.53972793, 0.09495186, 0.44267157, 0.72112119]])
"""
if _in_eager_mode():
if core._in_eager_mode():
from paddle.tensor.to_string import eager_tensor_to_string
return eager_tensor_to_string(self)
else:
......@@ -619,7 +619,7 @@ def monkey_patch_varbase():
raise RuntimeError(
"Only Leaf Tensor support the deepcopy at the moment, non-Leaf Tensors contains graph information that does't support deepcopy"
)
if _in_eager_mode():
if core._in_eager_mode():
new_varbase = core.eager.EagerTensor()
else:
new_varbase = core.VarBase()
......@@ -763,6 +763,14 @@ def monkey_patch_varbase():
else:
return None
@framework.dygraph_only
def _set_grad_ivar(self, value):
if isinstance(self, EagerParamBase):
self.grad = value
else:
raise TypeError(
"_set_grad_ivar is only supported for Parameter Tensor")
@framework.dygraph_only
def clear_gradient(self, set_to_zero=True):
if set_to_zero:
......@@ -770,6 +778,10 @@ def monkey_patch_varbase():
else:
self._clear_gradient()
@framework.dygraph_only
def clone(self):
return _C_ops_.assign(self)
if core._in_eager_mode() and not hasattr(core, "eager"):
return
......@@ -790,7 +802,9 @@ def monkey_patch_varbase():
if core._in_eager_mode():
setattr(core.eager.EagerTensor, "_grad_ivar", _grad_ivar)
setattr(core.eager.EagerTensor, "_set_grad_ivar", _set_grad_ivar)
setattr(core.eager.EagerTensor, "clear_gradient", clear_gradient)
setattr(core.eager.EagerTensor, "clone", clone)
else:
setattr(core.VarBase, "__name__", "Tensor")
setattr(core.VarBase, "grad", grad)
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册