未验证 提交 afafb1c3 编写于 作者: 0 0x45f 提交者: GitHub

Refine partial_program for new run_program OP (#40355)

* refine partial_program

* fix code for test_mnist.py train

* support quantify UT

* make __fake_vars and _double_grads to lazy

* fix comments
上级 1b0cecb7
...@@ -57,6 +57,7 @@ inline void run_program_dygraph_function( ...@@ -57,6 +57,7 @@ inline void run_program_dygraph_function(
auto grad_node = std::make_shared<GradNodeRunProgram>(1, 2); auto grad_node = std::make_shared<GradNodeRunProgram>(1, 2);
grad_node->SetFwdOutNames(out_names); grad_node->SetFwdOutNames(out_names);
grad_node->SetOut(out);
// Set Attributes // Set Attributes
grad_node->SetAttrMap(attrs); grad_node->SetAttrMap(attrs);
// Set TensorWrappers // Set TensorWrappers
......
...@@ -260,9 +260,9 @@ inline void RunProgramAPI( ...@@ -260,9 +260,9 @@ inline void RunProgramAPI(
} }
VLOG(2) << "The number of sub scopes after forward: " VLOG(2) << "The number of sub scopes after forward: "
<< out_scope_vec->front()->kids().size(); << out_scope_vec->front()->kids().size();
// #ifdef PADDLE_WITH_MKLDNN #ifdef PADDLE_WITH_MKLDNN
// if (FLAGS_use_mkldnn) paddle::platform::DontClearMKLDNNCache(place); if (FLAGS_use_mkldnn) paddle::platform::DontClearMKLDNNCache(place);
// #endif #endif
} }
inline void RunProgramGradAPI( inline void RunProgramGradAPI(
...@@ -357,7 +357,7 @@ inline void RunProgramGradAPI( ...@@ -357,7 +357,7 @@ inline void RunProgramGradAPI(
details::ShareTensorsFromScope(params_grad, *global_block, &scope); details::ShareTensorsFromScope(params_grad, *global_block, &scope);
// Step5. drop current scope // Step5. drop current scope
// global_inner_scope->DeleteScope(&scope); global_inner_scope->DeleteScope(&scope);
VLOG(2) << "The number of sub scopes after backward: " VLOG(2) << "The number of sub scopes after backward: "
<< global_inner_scope->kids().size(); << global_inner_scope->kids().size();
} }
...@@ -400,6 +400,10 @@ class GradNodeRunProgram : public egr::GradNodeBase { ...@@ -400,6 +400,10 @@ class GradNodeRunProgram : public egr::GradNodeBase {
paddle::platform::errors::InvalidArgument( paddle::platform::errors::InvalidArgument(
"The grads[0].size() and fwd_out_names_.size() should be equal.")); "The grads[0].size() and fwd_out_names_.size() should be equal."));
for (size_t i = 0; i < fwd_out_names_.size(); ++i) { for (size_t i = 0; i < fwd_out_names_.size(); ++i) {
auto &out_grad = egr::EagerUtils::unsafe_autograd_meta(*out_[i])->Grad();
const_cast<paddle::experimental::Tensor &>(out_grad).set_impl(
grads[0][i].impl());
const_cast<paddle::experimental::Tensor &>(grads[0][i]) const_cast<paddle::experimental::Tensor &>(grads[0][i])
.set_name(fwd_out_names_[i] + "@GRAD"); .set_name(fwd_out_names_[i] + "@GRAD");
} }
...@@ -432,6 +436,10 @@ class GradNodeRunProgram : public egr::GradNodeBase { ...@@ -432,6 +436,10 @@ class GradNodeRunProgram : public egr::GradNodeBase {
fwd_out_names_ = out_names; fwd_out_names_ = out_names;
} }
void SetOut(const std::vector<paddle::experimental::Tensor *> &out) {
out_ = out;
}
protected: protected:
void ConstructGradTensors( void ConstructGradTensors(
const std::vector<paddle::experimental::Tensor> &fwd_tensors, const std::vector<paddle::experimental::Tensor> &fwd_tensors,
...@@ -440,7 +448,11 @@ class GradNodeRunProgram : public egr::GradNodeBase { ...@@ -440,7 +448,11 @@ class GradNodeRunProgram : public egr::GradNodeBase {
// such as: name, tensor type(DenseTensor or SelectedRows). // such as: name, tensor type(DenseTensor or SelectedRows).
VLOG(3) << "fwd_tensors.size(): " << fwd_tensors.size(); VLOG(3) << "fwd_tensors.size(): " << fwd_tensors.size();
for (auto &fwd_t : fwd_tensors) { for (auto &fwd_t : fwd_tensors) {
grad_tensors->emplace_back(fwd_t.impl()); if (phi::DenseTensor::classof(fwd_t.impl().get())) {
grad_tensors->emplace_back(std::make_shared<phi::DenseTensor>());
} else if (phi::SelectedRows::classof(fwd_t.impl().get())) {
grad_tensors->emplace_back(std::make_shared<phi::SelectedRows>());
}
auto &grad_t = grad_tensors->back(); auto &grad_t = grad_tensors->back();
grad_t.set_name(fwd_t.name() + "@GRAD"); grad_t.set_name(fwd_t.name() + "@GRAD");
} }
...@@ -462,6 +474,7 @@ class GradNodeRunProgram : public egr::GradNodeBase { ...@@ -462,6 +474,7 @@ class GradNodeRunProgram : public egr::GradNodeBase {
std::vector<paddle::framework::Scope *> step_scope_; std::vector<paddle::framework::Scope *> step_scope_;
std::vector<std::string> fwd_out_names_; std::vector<std::string> fwd_out_names_;
std::vector<paddle::experimental::Tensor *> out_;
// Attribute Map // Attribute Map
paddle::framework::AttributeMap attrs_; paddle::framework::AttributeMap attrs_;
......
...@@ -31,7 +31,6 @@ static PyObject *eager_api_run_program(PyObject *self, PyObject *args, ...@@ -31,7 +31,6 @@ static PyObject *eager_api_run_program(PyObject *self, PyObject *args,
tstate = PyEval_SaveThread(); tstate = PyEval_SaveThread();
run_program_dygraph_function(X, Params, Out, OutScope, DOut, attrs); run_program_dygraph_function(X, Params, Out, OutScope, DOut, attrs);
std::cout << "end run_program_dygraph_function" << std::endl;
PyEval_RestoreThread(tstate); PyEval_RestoreThread(tstate);
tstate = nullptr; tstate = nullptr;
} catch (...) { } catch (...) {
......
...@@ -688,6 +688,21 @@ static PyObject* tensor_register_reduce_hook(TensorObject* self, PyObject* args, ...@@ -688,6 +688,21 @@ static PyObject* tensor_register_reduce_hook(TensorObject* self, PyObject* args,
EAGER_CATCH_AND_THROW_RETURN_NULL EAGER_CATCH_AND_THROW_RETURN_NULL
} }
static PyObject* set_grad_type(TensorObject* self, PyObject* args,
PyObject* kwargs) {
EAGER_TRY
auto var_type = pybind::CastPyArg2ProtoType(PyTuple_GET_ITEM(args, 0), 0);
auto grad_tensor =
egr::EagerUtils::unsafe_autograd_meta(self->tensor)->Grad();
if (var_type == framework::proto::VarType::LOD_TENSOR) {
grad_tensor.set_impl(std::make_shared<phi::DenseTensor>());
} else if (var_type == framework::proto::VarType::SELECTED_ROWS) {
grad_tensor.set_impl(std::make_shared<phi::SelectedRows>());
}
return Py_None;
EAGER_CATCH_AND_THROW_RETURN_NULL
}
PyMethodDef variable_methods[] = { PyMethodDef variable_methods[] = {
{"numpy", (PyCFunction)(void (*)(void))tensor_method_numpy, {"numpy", (PyCFunction)(void (*)(void))tensor_method_numpy,
METH_VARARGS | METH_KEYWORDS, NULL}, METH_VARARGS | METH_KEYWORDS, NULL},
...@@ -734,6 +749,8 @@ PyMethodDef variable_methods[] = { ...@@ -734,6 +749,8 @@ PyMethodDef variable_methods[] = {
{"_register_backward_hook", {"_register_backward_hook",
(PyCFunction)(void (*)(void))tensor_register_reduce_hook, (PyCFunction)(void (*)(void))tensor_register_reduce_hook,
METH_VARARGS | METH_KEYWORDS, NULL}, METH_VARARGS | METH_KEYWORDS, NULL},
{"_set_grad_type", (PyCFunction)(void (*)(void))set_grad_type,
METH_VARARGS | METH_KEYWORDS, NULL},
{NULL, NULL, 0, NULL}}; {NULL, NULL, 0, NULL}};
} // namespace pybind } // namespace pybind
......
...@@ -104,7 +104,7 @@ class FunctionSpec(object): ...@@ -104,7 +104,7 @@ class FunctionSpec(object):
if isinstance(input_var, np.ndarray): if isinstance(input_var, np.ndarray):
input_var = paddle.static.InputSpec.from_numpy(input_var) input_var = paddle.static.InputSpec.from_numpy(input_var)
_set_spec_stop_gradient(input_var, True) _set_spec_stop_gradient(input_var, True)
elif isinstance(input_var, core.VarBase): elif isinstance(input_var, (core.VarBase, core.eager.Tensor)):
stop_gradient = input_var.stop_gradient stop_gradient = input_var.stop_gradient
input_var = paddle.static.InputSpec.from_tensor(input_var) input_var = paddle.static.InputSpec.from_tensor(input_var)
_set_spec_stop_gradient(input_var, stop_gradient) _set_spec_stop_gradient(input_var, stop_gradient)
......
...@@ -148,10 +148,7 @@ class PartialProgramLayer: ...@@ -148,10 +148,7 @@ class PartialProgramLayer:
self._origin_main_program = self._verify_program(main_program) self._origin_main_program = self._verify_program(main_program)
self._tmp_scope_vec = self._create_scope_vec() self._tmp_scope_vec = self._create_scope_vec()
# A fake_var to handle empty input or output
self.__fake_vars = _create_fake_var()
# Set default mode to train # Set default mode to train
self._double_grads = self._get_double_grads(self._origin_main_program)
self.training = True self.training = True
custom_white_list, custom_black_list = None, None custom_white_list, custom_black_list = None, None
...@@ -163,6 +160,14 @@ class PartialProgramLayer: ...@@ -163,6 +160,14 @@ class PartialProgramLayer:
custom_white_list=custom_white_list, custom_white_list=custom_white_list,
custom_black_list=custom_black_list) custom_black_list=custom_black_list)
@LazyInitialized
def __fake_vars(self):
return _create_fake_var()
@LazyInitialized
def _double_grads(self):
return self._get_double_grads(self._origin_main_program)
@LazyInitialized @LazyInitialized
def _infer_program(self): def _infer_program(self):
""" """
...@@ -356,8 +361,10 @@ class PartialProgramLayer: ...@@ -356,8 +361,10 @@ class PartialProgramLayer:
def drop_scope_if_no_grad(self): def drop_scope_if_no_grad(self):
tracer = framework._dygraph_tracer() tracer = framework._dygraph_tracer()
scope = self._tmp_scope_vec.value().get_scope() if isinstance(
self._tmp_scope_vec, (core.VarBase)) else self._tmp_scope_vec[0]
if self.training and not tracer._has_grad: if self.training and not tracer._has_grad:
self._tmp_scope_vec.value().get_scope().drop_kids() scope.drop_kids()
@property @property
def program(self): def program(self):
...@@ -449,18 +456,14 @@ class PartialProgramLayer: ...@@ -449,18 +456,14 @@ class PartialProgramLayer:
def _create_scope_vec(self): def _create_scope_vec(self):
# Hold forward variables # Hold forward variables
tmp_scope_vec = None tmp_scope_vec = None
inner_scope = core.Scope()
if not core._in_eager_mode(): if not core._in_eager_mode():
tmp_scope_vec = core.VarBase(core.VarDesc.VarType.FP32, [], tmp_scope_vec = core.VarBase(core.VarDesc.VarType.FP32, [],
"program_out_scope", "program_out_scope",
core.VarDesc.VarType.STEP_SCOPES, True) core.VarDesc.VarType.STEP_SCOPES, True)
# TODO(jiabin): Support this later.
# else:
# tmp_scope_vec = core.eager.Tensor(core.VarDesc.VarType.FP32, [],
# "program_out_scope",
# core.VarDesc.VarType.STEP_SCOPES, True)
inner_scope = core.Scope()
tmp_scope_vec.value().set_scope(inner_scope) tmp_scope_vec.value().set_scope(inner_scope)
else:
tmp_scope_vec = [inner_scope]
return tmp_scope_vec return tmp_scope_vec
def _restore_out(self, out_vars): def _restore_out(self, out_vars):
...@@ -598,12 +601,10 @@ def _create_fake_var(): ...@@ -598,12 +601,10 @@ def _create_fake_var():
core.VarDesc.VarType.RAW, False) core.VarDesc.VarType.RAW, False)
] ]
else: else:
return [] return [
# TODO(jiabin): Support this later core.eager.Tensor(core.VarDesc.VarType.FP32, [], "Fake_var",
# return [ core.VarDesc.VarType.RAW, False)
# core.eager.Tensor(core.VarDesc.VarType.FP32, [], "Fake_var", ]
# core.VarDesc.VarType.RAW, False)
# ]
def partial_program_from(concrete_program): def partial_program_from(concrete_program):
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册