From 432ac70124b802ca9a18900a3d1a92b73c8bae99 Mon Sep 17 00:00:00 2001 From: Zeng Jinle <32832641+sneaxiy@users.noreply.github.com> Date: Mon, 27 May 2019 21:13:57 +0800 Subject: [PATCH] clean code of py_layer in dygraph mode,test=develop (#17661) --- paddle/fluid/imperative/layer.cc | 206 ++++++------------ paddle/fluid/imperative/layer.h | 30 --- paddle/fluid/imperative/tracer.cc | 51 ----- paddle/fluid/imperative/tracer.h | 3 - paddle/fluid/pybind/imperative.cc | 182 ++++++++++++---- paddle/fluid/pybind/imperative.h | 23 -- paddle/fluid/pybind/pybind.cc | 130 ----------- python/paddle/fluid/dygraph/layers.py | 75 +------ .../tests/unittests/test_imperative_basic.py | 83 ------- 9 files changed, 210 insertions(+), 573 deletions(-) diff --git a/paddle/fluid/imperative/layer.cc b/paddle/fluid/imperative/layer.cc index cfe11075d19..4682196c6d7 100644 --- a/paddle/fluid/imperative/layer.cc +++ b/paddle/fluid/imperative/layer.cc @@ -33,11 +33,6 @@ namespace paddle { namespace imperative { -const char* PyLayer::kFwdInp = "X"; -const char* PyLayer::kFwdOut = "Out"; - -std::map py_funcs_; - using framework::Variable; namespace detail { @@ -255,94 +250,85 @@ framework::LoDTensor& VarBase::GradValue() { std::map> OpBase::ApplyGrad( BackwardSumMap* bck_map, GradientRef* grad_ref, const detail::BackwardStrategy& bck_stratedy) { - PADDLE_ENFORCE(!grad_op_descs_.empty() || backward_id_ > 0, - "%s has no backward implementation", Type()); + PADDLE_ENFORCE(!grad_op_descs_.empty(), "%s has no backward implementation", + Type()); VLOG(3) << "apply op grad: " << Type(); std::vector tmp_grad_outputs; - if (backward_id_ > 0) { - VLOG(3) << "py_layer_grad"; - tmp_grad_outputs.resize(1); - tmp_grad_outputs[0][framework::GradVarName(PyLayer::kFwdOut)] = - PyLayer::ApplyGrad( - backward_id_, - grad_input_vars_[0][framework::GradVarName(PyLayer::kFwdInp)]); - } else { - const size_t grad_op_count = grad_op_descs_.size(); - - tmp_grad_outputs.resize(grad_op_count); - for (size_t k = 0; k < grad_op_count; ++k) { - framework::OpDesc* grad_op_desc = grad_op_descs_[k]; - platform::RecordEvent record_event(grad_op_desc->Type()); - auto& grad_output_variable_map = grad_output_vars_[k]; - VLOG(3) << "apply grad op " << grad_op_desc->Type(); - - // Allocate tmp grad output variable - for (const auto& it : grad_output_variable_map) { - auto& outputs = tmp_grad_outputs[k][it.first]; - outputs.reserve(it.second.size()); - for (size_t i = 0; i < it.second.size(); ++i) { - VarBase* origin_grad_var_base = it.second[i]; - - // Allocate a new variable - VarBase* tmp_grad_var_base = new VarBase( - string::Sprintf("%s@IGrad", origin_grad_var_base->Name()), - origin_grad_var_base->DataType(), origin_grad_var_base->Dims(), - place_, true, false); - outputs.emplace_back(tmp_grad_var_base); - } + const size_t grad_op_count = grad_op_descs_.size(); + + tmp_grad_outputs.resize(grad_op_count); + for (size_t k = 0; k < grad_op_count; ++k) { + framework::OpDesc* grad_op_desc = grad_op_descs_[k]; + platform::RecordEvent record_event(grad_op_desc->Type()); + auto& grad_output_variable_map = grad_output_vars_[k]; + VLOG(3) << "apply grad op " << grad_op_desc->Type(); + + // Allocate tmp grad output variable + for (const auto& it : grad_output_variable_map) { + auto& outputs = tmp_grad_outputs[k][it.first]; + outputs.reserve(it.second.size()); + for (size_t i = 0; i < it.second.size(); ++i) { + VarBase* origin_grad_var_base = it.second[i]; + + // Allocate a new variable + VarBase* tmp_grad_var_base = new VarBase( + string::Sprintf("%s@IGrad", origin_grad_var_base->Name()), + origin_grad_var_base->DataType(), origin_grad_var_base->Dims(), + place_, true, false); + outputs.emplace_back(tmp_grad_var_base); } + } - // No need to do compile time infer shape here. - // grad_op_desc_->InferShape(*block_); - // grad_op_desc->InferVarType(block_); + // No need to do compile time infer shape here. + // grad_op_desc_->InferShape(*block_); + // grad_op_desc->InferVarType(block_); - std::unique_ptr opbase = - framework::OpRegistry::CreateOp(*grad_op_desc); + std::unique_ptr opbase = + framework::OpRegistry::CreateOp(*grad_op_desc); - auto& info = framework::OpInfoMap::Instance().Get(grad_op_desc->Type()); - if (info.infer_var_type_) { - RuntimeInferVarTypeContext infer_var_type_ctx( - &grad_input_vars_[k], &tmp_grad_outputs[k], &attrs_); - info.infer_var_type_(&infer_var_type_ctx); - } + auto& info = framework::OpInfoMap::Instance().Get(grad_op_desc->Type()); + if (info.infer_var_type_) { + RuntimeInferVarTypeContext infer_var_type_ctx( + &grad_input_vars_[k], &tmp_grad_outputs[k], &attrs_); + info.infer_var_type_(&infer_var_type_ctx); + } - framework::OperatorWithKernel* op_kernel = - dynamic_cast(opbase.get()); - PADDLE_ENFORCE_NOT_NULL(op_kernel, "only support op with kernel"); + framework::OperatorWithKernel* op_kernel = + dynamic_cast(opbase.get()); + PADDLE_ENFORCE_NOT_NULL(op_kernel, "only support op with kernel"); - // Run grad op - framework::VariableValueMap grad_invars_map; - framework::VariableValueMap grad_outvars_map; + // Run grad op + framework::VariableValueMap grad_invars_map; + framework::VariableValueMap grad_outvars_map; - for (const auto& it : grad_input_vars_[k]) { - auto& grad_invars = grad_invars_map[it.first]; - grad_invars.reserve(it.second.size()); - for (const VarBase* grad_inp : it.second) { - PADDLE_ENFORCE_NOT_NULL(grad_inp->var_, "op %s input %s nullptr", - grad_op_desc->Type(), grad_inp->Name()); + for (const auto& it : grad_input_vars_[k]) { + auto& grad_invars = grad_invars_map[it.first]; + grad_invars.reserve(it.second.size()); + for (const VarBase* grad_inp : it.second) { + PADDLE_ENFORCE_NOT_NULL(grad_inp->var_, "op %s input %s nullptr", + grad_op_desc->Type(), grad_inp->Name()); - grad_invars.emplace_back(grad_inp->var_.get()); - } + grad_invars.emplace_back(grad_inp->var_.get()); } + } - for (const auto& it : tmp_grad_outputs[k]) { - auto& grad_outvars = grad_outvars_map[it.first]; - grad_outvars.reserve(it.second.size()); - for (VarBase* grad_out : it.second) { - PADDLE_ENFORCE_NOT_NULL(grad_out->var_, "op %s output %s nullptr", - grad_op_desc->Type(), grad_out->Name()); + for (const auto& it : tmp_grad_outputs[k]) { + auto& grad_outvars = grad_outvars_map[it.first]; + grad_outvars.reserve(it.second.size()); + for (VarBase* grad_out : it.second) { + PADDLE_ENFORCE_NOT_NULL(grad_out->var_, "op %s output %s nullptr", + grad_op_desc->Type(), grad_out->Name()); - grad_outvars.emplace_back(grad_out->var_.get()); - } + grad_outvars.emplace_back(grad_out->var_.get()); } - - framework::RuntimeContext ctx(grad_invars_map, grad_outvars_map); - framework::Scope scope; - PreparedOp p = PreparedOp::Prepare(ctx, *op_kernel, place_); - p.op.RuntimeInferShape(scope, place_, ctx); - p.func( - framework::ExecutionContext(p.op, scope, *p.dev_ctx, p.ctx, nullptr)); } + + framework::RuntimeContext ctx(grad_invars_map, grad_outvars_map); + framework::Scope scope; + PreparedOp p = PreparedOp::Prepare(ctx, *op_kernel, place_); + p.op.RuntimeInferShape(scope, place_, ctx); + p.func( + framework::ExecutionContext(p.op, scope, *p.dev_ctx, p.ctx, nullptr)); } platform::RecordEvent record_event("merge_grads"); @@ -439,69 +425,5 @@ void VarBase::RunBackward(const detail::BackwardStrategy& bck_stratedy) { Autograd().RunBackward(this, bck_stratedy); } -void PyLayer::RegisterFunc(int func_id, const py::object& py_func) { - py_funcs_[func_id] = py_func; -} - -int PyLayer::NumFuncs() { return py_funcs_.size(); } - -std::vector> PyLayer::Apply( - int func_id, const std::vector& inputs) { - PADDLE_ENFORCE(py_funcs_.find(func_id) != py_funcs_.end()); - return CallPythonFunc(py_funcs_[func_id], inputs); -} - -std::vector PyLayer::ApplyGrad(int func_id, - const std::vector& inputs) { - PADDLE_ENFORCE(py_funcs_.find(func_id) != py_funcs_.end()); - auto rets = CallPythonFunc(py_funcs_[func_id], inputs); - - std::vector outs; - outs.reserve(rets.size()); - for (size_t i = 0U; i != rets.size(); ++i) { - outs.emplace_back(new VarBase( - string::Sprintf("%s_out_%d", framework::GradVarName(PyLayer::kFwdOut), - i), - std::move(rets[i]), nullptr, true)); - } - - return outs; -} - -std::vector> PyLayer::CallPythonFunc( - const py::object& callable, const std::vector& ins) { - py::gil_scoped_acquire guard; - py::tuple in_args(ins.size()); - for (size_t i = 0; i < ins.size(); ++i) { - const framework::LoDTensor& t = ins[i]->var_->Get(); - in_args[i] = t.IsInitialized() ? py::cast(t) : py::cast(nullptr); - } - VLOG(3) << "pyfunc in " << py::len(in_args); - - // TODO(panyx0718): Who owns the returned LoDTensor. - auto ret = callable(in_args); - auto ret_tuple = py::cast(ret); - size_t ret_num = py::len(ret_tuple); - std::vector> outs; - outs.reserve(ret_num); - VLOG(3) << "pyfunc out " << ret_num; - for (size_t i = 0; i < ret_num; ++i) { - try { - auto* py_out_tensor = py::cast(ret_tuple[i]); - PADDLE_ENFORCE_NOT_NULL(py_out_tensor, - "Output tensor %d should not be nullptr", i); - auto var = - std::unique_ptr(new framework::Variable()); - auto* tensor = var->GetMutable(); - tensor->ShareDataWith(*py_out_tensor); - tensor->set_lod(py_out_tensor->lod()); - outs.emplace_back(std::move(var)); - } catch (py::cast_error&) { - PADDLE_THROW("The %d-th output must be LoDTensor", i); - } - } - return outs; -} - } // namespace imperative } // namespace paddle diff --git a/paddle/fluid/imperative/layer.h b/paddle/fluid/imperative/layer.h index cfd43ef6ce3..92189db9f71 100644 --- a/paddle/fluid/imperative/layer.h +++ b/paddle/fluid/imperative/layer.h @@ -279,8 +279,6 @@ class PYBIND11_HIDDEN OpBase { OpBase(const std::string& type) : type_(type), trace_id_(-1), - forward_id_(-1), - backward_id_(-1), place_(platform::CPUPlace()), backward_hooks_() {} @@ -335,16 +333,10 @@ class PYBIND11_HIDDEN OpBase { } std::string type_; - // One of `trace_id_` or `forward_id_` is set, not both. - // For pure python PyLayer, use `forward_id_`, otherwise, use trace_id_. int trace_id_; - int forward_id_; - // When has backward, one of `grad_op_descs_` or `backward_id_` is set, - // not both. // Note: each fwd op corresponds to a vector of bwd ops. std::vector grad_op_descs_; - int backward_id_; platform::Place place_; @@ -373,28 +365,6 @@ class Layer { } }; -class PyLayer { - public: - virtual ~PyLayer() {} - - static const char* kFwdInp; - static const char* kFwdOut; - - static void RegisterFunc(int func_id, const py::object& py_func); - - static int NumFuncs(); - - static std::vector> Apply( - int func_id, const std::vector& inputs); - - static std::vector ApplyGrad(int func_id, - const std::vector& inputs); - - private: - static std::vector> CallPythonFunc( - const py::object& callable, const std::vector& ins); -}; - // infer var type context for imperative mode class PYBIND11_HIDDEN RuntimeInferVarTypeContext : public framework::InferVarTypeContext { diff --git a/paddle/fluid/imperative/tracer.cc b/paddle/fluid/imperative/tracer.cc index 45ae15e2ede..1fc490b889c 100644 --- a/paddle/fluid/imperative/tracer.cc +++ b/paddle/fluid/imperative/tracer.cc @@ -290,56 +290,5 @@ std::set Tracer::Trace(OpBase* op, const VarBasePtrMap& inputs, return vars_saved_for_backward; } -std::vector Tracer::PyTrace(OpBase* op, - const std::vector& inputs, - bool stop_gradient) { - VLOG(3) << "py_trace " << op->Type(); - - op->input_vars_[PyLayer::kFwdInp] = inputs; - - std::vector> ret_vars = - PyLayer::Apply(op->forward_id_, inputs); - op->TrackPreOp(PyLayer::kFwdInp, inputs); - - std::vector& outputs = op->output_vars_[PyLayer::kFwdOut]; - outputs.reserve(ret_vars.size()); - for (size_t i = 0U; i != ret_vars.size(); ++i) { - VarBase* out = new VarBase(string::Sprintf("%s_out_%d", op->Type(), i), - std::move(ret_vars[i]), nullptr, stop_gradient); - outputs.emplace_back(out); - out->TrackPreOp(op, PyLayer::kFwdOut, i, stop_gradient); - } - - if (!stop_gradient) { - VLOG(5) << "start construct backward op"; - op->grad_input_vars_.resize(1); - op->grad_output_vars_.resize(1); - auto& grad_input_vars = - op->grad_input_vars_[0][framework::GradVarName(PyLayer::kFwdInp)]; - auto& grad_output_vars = - op->grad_output_vars_[0][framework::GradVarName(PyLayer::kFwdOut)]; - - for (VarBase* inp : inputs) { - grad_input_vars.push_back(inp); - } - for (VarBase* out : outputs) { - grad_input_vars.push_back(out); - } - - // TODO(minqiyang): Add GPU support for PyLayer, only support CPU now - platform::CPUPlace place; - for (VarBase* out : outputs) { - InitGrad(out, platform::DeviceContextPool::Instance().Get(place)); - grad_input_vars.push_back(out->grads_); - } - - for (VarBase* inp : inputs) { - InitGrad(inp, platform::DeviceContextPool::Instance().Get(place)); - grad_output_vars.push_back(inp->grads_); - } - } - return outputs; -} - } // namespace imperative } // namespace paddle diff --git a/paddle/fluid/imperative/tracer.h b/paddle/fluid/imperative/tracer.h index a87f3b8009d..9d95e3cd07d 100644 --- a/paddle/fluid/imperative/tracer.h +++ b/paddle/fluid/imperative/tracer.h @@ -53,9 +53,6 @@ class Tracer { const platform::Place expected_place, const bool stop_gradient = false); - std::vector PyTrace(OpBase* op, const std::vector& inputs, - bool stop_gradient = false); - private: platform::Place GetPlace(const VarBasePtrMap& inputs); diff --git a/paddle/fluid/pybind/imperative.cc b/paddle/fluid/pybind/imperative.cc index 265707f1bcc..5110d5e40d4 100644 --- a/paddle/fluid/pybind/imperative.cc +++ b/paddle/fluid/pybind/imperative.cc @@ -18,8 +18,11 @@ limitations under the License. */ #include #include #include +#include #include "paddle/fluid/framework/block_desc.h" +#include "paddle/fluid/imperative/layer.h" +#include "paddle/fluid/imperative/profiler.h" #include "paddle/fluid/imperative/tracer.h" #include "paddle/fluid/imperative/type_defs.h" @@ -28,77 +31,182 @@ limitations under the License. */ namespace paddle { namespace pybind { +class Layer : public imperative::Layer { + public: + using imperative::Layer::Layer; // Inherit constructors + + std::vector Forward( + const std::vector &inputs) override { + PYBIND11_OVERLOAD(std::vector, Layer, Forward, + inputs); // NOLINT + } +}; + +class PYBIND11_HIDDEN PyOpBase : public imperative::OpBase { + public: + using imperative::OpBase::OpBase; // Inherit constructors + + PyOpBase(const std::string &name) : OpBase(name) {} +}; + // Bind Methods -void BindImperative(pybind11::module* m) { - pybind11::class_(*m, "Tracer", "") +void BindImperative(pybind11::module *m_ptr) { + namespace py = ::pybind11; + + auto &m = *m_ptr; + + py::class_ backward_strategy( + m, "BackwardStrategy", R"DOC()DOC"); + backward_strategy.def(py::init()) + .def_property("sort_sum_gradient", + [](const imperative::detail::BackwardStrategy &self) { + return self.sorted_sum_gradient_; + }, + [](imperative::detail::BackwardStrategy &self, + bool sorted_sum_gradient) { + self.sorted_sum_gradient_ = sorted_sum_gradient; + }); + + m.def("start_imperative_gperf_profiler", + []() { imperative::StartProfile(); }); + + m.def("stop_imperative_gperf_profiler", []() { imperative::StopProfile(); }); + + py::class_(m, "VarBase", R"DOC()DOC") + .def( + py::init, const paddle::platform::CPUPlace, + bool, bool>()) + .def( + py::init, + const paddle::platform::CUDAPlace, bool, bool>()) + .def("_run_backward", + [](imperative::VarBase &self, + const imperative::detail::BackwardStrategy &bckst) { + self.RunBackward(bckst); + }) + .def("_grad_name", &imperative::VarBase::GradName) + .def("_grad_value", &imperative::VarBase::GradValue) + .def("_clear_gradient", &imperative::VarBase::ClearGradient) + .def("_grad_ivar", + [](const imperative::VarBase &self) { return self.grads_; }, + py::return_value_policy::reference) + .def("_copy_to", + [](const imperative::VarBase &self, const platform::CPUPlace &place, + bool blocking) { + return self.NewVarBase(place, blocking).release(); + }, + py::return_value_policy::take_ownership) + .def("_copy_to", + [](const imperative::VarBase &self, const platform::CUDAPlace &place, + bool blocking) { + return self.NewVarBase(place, blocking).release(); + }, + py::return_value_policy::take_ownership) + .def("value", + [](const imperative::VarBase &self) { return self.var_.get(); }, + py::return_value_policy::reference) + .def_property("name", &imperative::VarBase::Name, + &imperative::VarBase::SetName) + .def_property_readonly("shape", &imperative::VarBase::Shape) + .def_property_readonly("dtype", &imperative::VarBase::DataType) + .def_property("persistable", &imperative::VarBase::IsPersistable, + &imperative::VarBase::SetPersistable) + .def_property("stop_gradient", &imperative::VarBase::IsStopGradient, + &imperative::VarBase::SetStopGradient); + + py::class_(m, "OpBase", R"DOC()DOC") + .def(py::init()) + .def("register_backward_hooks", + [](imperative::OpBase &self, const py::object &callable) { + self.RegisterBackwardHooks(callable); + }) + .def_property("_trace_id", + [](const imperative::OpBase &self) { + py::gil_scoped_release release; + return self.trace_id_; + }, + [](imperative::OpBase &self, int trace_id) { + py::gil_scoped_release release; + self.trace_id_ = trace_id; + }, + py::return_value_policy::reference) + .def_property_readonly("type", &imperative::OpBase::Type); + + py::class_ layer(m, "Layer"); + layer.def(py::init<>()) + .def("forward", [](imperative::Layer &self, + const std::vector &inputs) { + return self.Forward(inputs); + }); + + py::class_(*m, "Tracer", "") .def("__init__", - [](imperative::Tracer& self, framework::BlockDesc* root_block) { + [](imperative::Tracer &self, framework::BlockDesc *root_block) { new (&self) imperative::Tracer(root_block); }) .def("trace", - [](imperative::Tracer& self, imperative::OpBase* op, - const imperative::VarBasePtrMap& inputs, - imperative::VarBasePtrMap* outputs, + [](imperative::Tracer &self, imperative::OpBase *op, + const imperative::VarBasePtrMap &inputs, + imperative::VarBasePtrMap *outputs, framework::AttributeMap attrs_map, const platform::CPUPlace expected_place, const bool stop_gradient = false) { - pybind11::gil_scoped_release release; - return self.Trace(op, inputs, outputs, attrs_map, expected_place, - stop_gradient); - }) - .def("trace", - [](imperative::Tracer& self, imperative::OpBase* op, - const imperative::VarBasePtrMap& inputs, - imperative::VarBasePtrMap* outputs, - framework::AttributeMap attrs_map, - const platform::CUDAPlace expected_place, - const bool stop_gradient = false) { - pybind11::gil_scoped_release release; + py::gil_scoped_release release; return self.Trace(op, inputs, outputs, attrs_map, expected_place, stop_gradient); }) - .def("py_trace", &imperative::Tracer::PyTrace, - pybind11::return_value_policy::take_ownership); + .def("trace", [](imperative::Tracer &self, imperative::OpBase *op, + const imperative::VarBasePtrMap &inputs, + imperative::VarBasePtrMap *outputs, + framework::AttributeMap attrs_map, + const platform::CUDAPlace expected_place, + const bool stop_gradient = false) { + py::gil_scoped_release release; + return self.Trace(op, inputs, outputs, attrs_map, expected_place, + stop_gradient); + }); // define parallel context - pybind11::class_ parallel_strategy( - *m, "ParallelStrategy", ""); - parallel_strategy.def(pybind11::init()) + py::class_ parallel_strategy( + m, "ParallelStrategy", ""); + parallel_strategy.def(py::init()) .def_property( "nranks", - [](const imperative::ParallelStrategy& self) { return self.nranks_; }, - [](imperative::ParallelStrategy& self, int nranks) { + [](const imperative::ParallelStrategy &self) { return self.nranks_; }, + [](imperative::ParallelStrategy &self, int nranks) { self.nranks_ = nranks; }) .def_property("local_rank", - [](const imperative::ParallelStrategy& self) { + [](const imperative::ParallelStrategy &self) { return self.local_rank_; }, - [](imperative::ParallelStrategy& self, int local_rank) { + [](imperative::ParallelStrategy &self, int local_rank) { self.local_rank_ = local_rank; }) .def_property( "trainer_endpoints", - [](const imperative::ParallelStrategy& self) { + [](const imperative::ParallelStrategy &self) { return self.trainer_endpoints_; }, - [](imperative::ParallelStrategy& self, std::vector eps) { + [](imperative::ParallelStrategy &self, std::vector eps) { self.trainer_endpoints_ = eps; }) .def_property("current_endpoint", - [](const imperative::ParallelStrategy& self) { + [](const imperative::ParallelStrategy &self) { return self.current_endpoint_; }, - [](imperative::ParallelStrategy& self, - const std::string& ep) { self.current_endpoint_ = ep; }); + [](imperative::ParallelStrategy &self, + const std::string &ep) { self.current_endpoint_ = ep; }); #if defined(PADDLE_WITH_CUDA) && !defined(_WIN32) - pybind11::class_ nccl_ctx( - *m, "NCCLParallelContext"); + py::class_ nccl_ctx(m, + "NCCLParallelContext"); nccl_ctx - .def(pybind11::init()) - .def("init", [](imperative::NCCLParallelContext& self) { self.Init(); }); + .def(py::init()) + .def("init", [](imperative::NCCLParallelContext &self) { self.Init(); }); #endif } diff --git a/paddle/fluid/pybind/imperative.h b/paddle/fluid/pybind/imperative.h index 5c5556501ff..cfe185bbfbb 100644 --- a/paddle/fluid/pybind/imperative.h +++ b/paddle/fluid/pybind/imperative.h @@ -24,29 +24,6 @@ limitations under the License. */ namespace paddle { namespace pybind { -class Layer : public imperative::Layer { - public: - using imperative::Layer::Layer; // Inherit constructors - - std::vector Forward( - const std::vector& inputs) override { - PYBIND11_OVERLOAD(std::vector, Layer, Forward, - inputs); // NOLINT - } -}; - -class PYBIND11_HIDDEN PyOpBase : public imperative::OpBase { - public: - using imperative::OpBase::OpBase; // Inherit constructors - - PyOpBase(const std::string& name) : OpBase(name) {} -}; - -class PyVarBase : public imperative::VarBase { - public: - using imperative::VarBase::VarBase; // Inherit constructors -}; - void BindImperative(pybind11::module* m); } // namespace pybind diff --git a/paddle/fluid/pybind/pybind.cc b/paddle/fluid/pybind/pybind.cc index 64c95a14eac..a9bfb99333d 100644 --- a/paddle/fluid/pybind/pybind.cc +++ b/paddle/fluid/pybind/pybind.cc @@ -39,8 +39,6 @@ limitations under the License. */ #include "paddle/fluid/framework/scope_pool.h" #include "paddle/fluid/framework/selected_rows.h" #include "paddle/fluid/framework/version.h" -#include "paddle/fluid/imperative/layer.h" -#include "paddle/fluid/imperative/profiler.h" #include "paddle/fluid/memory/allocation/allocator_strategy.h" #include "paddle/fluid/memory/allocation/legacy_allocator.h" #include "paddle/fluid/operators/activation_op.h" @@ -190,134 +188,6 @@ PYBIND11_MODULE(core, m) { m.def("print_mem_usage", []() { return memory::allocation::GPUMemMonitor.PrintMemUsage(); }); - py::class_ backward_strategy( - m, "BackwardStrategy", R"DOC()DOC"); - backward_strategy.def(py::init()) - .def_property("sort_sum_gradient", - [](const imperative::detail::BackwardStrategy &self) { - return self.sorted_sum_gradient_; - }, - [](imperative::detail::BackwardStrategy &self, - bool sorted_sum_gradient) { - self.sorted_sum_gradient_ = sorted_sum_gradient; - }); - - m.def("start_imperative_gperf_profiler", - []() { imperative::StartProfile(); }); - - m.def("stop_imperative_gperf_profiler", []() { imperative::StopProfile(); }); - - py::class_(m, "VarBase", R"DOC()DOC") - .def( - py::init, const paddle::platform::CPUPlace, - bool, bool>()) - .def( - py::init, - const paddle::platform::CUDAPlace, bool, bool>()) - .def("_run_backward", - [](imperative::VarBase &self, - const imperative::detail::BackwardStrategy &bckst) { - self.RunBackward(bckst); - }) - .def("_grad_name", &imperative::VarBase::GradName) - .def("_grad_value", &imperative::VarBase::GradValue) - .def("_clear_gradient", &imperative::VarBase::ClearGradient) - .def("_grad_ivar", - [](const imperative::VarBase &self) { return self.grads_; }, - py::return_value_policy::reference) - .def("_copy_to", - [](const imperative::VarBase &self, const platform::CPUPlace &place, - bool blocking) { - std::unique_ptr new_var = - self.NewVarBase(place, blocking); - return new_var.release(); - }, - py::return_value_policy::take_ownership) - .def("_copy_to", - [](const imperative::VarBase &self, const platform::CUDAPlace &place, - bool blocking) { - std::unique_ptr new_var = - self.NewVarBase(place, blocking); - return new_var.release(); - }, - py::return_value_policy::take_ownership) - .def("value", - [](const imperative::VarBase &self) { return self.var_.get(); }, - py::return_value_policy::reference) - .def_property("name", &imperative::VarBase::Name, - &imperative::VarBase::SetName) - .def_property_readonly("shape", &imperative::VarBase::Shape) - .def_property_readonly("dtype", &imperative::VarBase::DataType) - .def_property("persistable", &imperative::VarBase::IsPersistable, - &imperative::VarBase::SetPersistable) - .def_property("stop_gradient", &imperative::VarBase::IsStopGradient, - &imperative::VarBase::SetStopGradient); - - py::class_(m, "OpBase", R"DOC()DOC") - .def(py::init()) - .def("register_backward_hooks", - [](imperative::OpBase &self, const py::object &callable) { - self.RegisterBackwardHooks(callable); - }) - .def_property("_trace_id", - [](const imperative::OpBase &self) { - pybind11::gil_scoped_release release; - return self.trace_id_; - }, - [](imperative::OpBase &self, int trace_id) { - pybind11::gil_scoped_release release; - self.trace_id_ = trace_id; - }, - py::return_value_policy::reference) - .def_property( - "forward_id", - [](const imperative::OpBase &self) { return self.forward_id_; }, - [](imperative::OpBase &self, int forward_id) { - self.forward_id_ = forward_id; - }, - py::return_value_policy::reference) - .def_property_readonly("type", &imperative::OpBase::Type) - .def_property( - "backward_id", - [](const imperative::OpBase &self) { return self.backward_id_; }, - [](imperative::OpBase &self, int backward_id) { - self.backward_id_ = backward_id; - }, - py::return_value_policy::reference); - - py::class_ layer(m, "Layer"); - layer.def(py::init<>()) - .def("forward", [](imperative::Layer &self, - const std::vector &inputs) { - return self.Forward(inputs); - }); - - py::class_(m, "PyLayer") - .def(py::init<>()) - .def_static( - "apply", - [](int func_id, const std::vector &inputs) - -> std::vector { - auto ret_vars = imperative::PyLayer::Apply(func_id, inputs); - std::vector outputs; - outputs.reserve(ret_vars.size()); - for (size_t i = 0U; i != ret_vars.size(); ++i) { - // TODO(minqiyang): use unique_name generator to set a name - outputs.emplace_back(new imperative::VarBase( - "", std::move(ret_vars[i]), nullptr, true)); - } - - return outputs; - }, - py::return_value_policy::take_ownership) - .def_static("register_func", - [](int func_id, const py::object &callable) { - imperative::PyLayer::RegisterFunc(func_id, callable); - }) - .def_static("num_funcs", &imperative::PyLayer::NumFuncs); - BindImperative(&m); py::class_(m, "Tensor", py::buffer_protocol()) diff --git a/python/paddle/fluid/dygraph/layers.py b/python/paddle/fluid/dygraph/layers.py index 54b34919eae..75d0caa4f74 100644 --- a/python/paddle/fluid/dygraph/layers.py +++ b/python/paddle/fluid/dygraph/layers.py @@ -25,7 +25,7 @@ from .layer_object_helper import LayerObjectHelper from paddle.fluid import framework from ..param_attr import ParamAttr -__all__ = ['Layer', 'PyLayer'] +__all__ = ['Layer'] class Layer(core.Layer): @@ -266,76 +266,3 @@ class Layer(core.Layer): for layer_name, layer_item in self._sub_layers.items(): if layer_item is not None: layer_item.load_dict(stat_dict) - - -class PyLayer(core.PyLayer): - """Layers composed of user-defined python codes.""" - - def __init__(self): - super(PyLayer, self).__init__() - - def train(self): - framework._dygraph_tracer().train_mode() - - def eval(self): - framework._dygraph_tracer().eval_mode() - - @classmethod - def _do_forward(cls, inputs): - return cls._to_tuple(cls.forward(inputs)) - - @classmethod - def _do_backward(cls, inputs): - return cls._to_tuple(cls.backward(inputs)) - - @staticmethod - def _to_tuple(inputs): - if not isinstance(inputs, list) and not isinstance(inputs, tuple): - inputs = [inputs] - ret = [] - for inp in inputs: - if isinstance(inp, core.LoDTensor): - ret.append(inp) - else: - tensor = core.LoDTensor() - tensor.set(inp, core.CPUPlace()) - ret.append(tensor) - return tuple(ret) - - @staticmethod - def forward(*inputs): - raise NotImplementedError - - @staticmethod - def backward(*douts): - raise NotImplementedError - - @classmethod - def __call__(cls, *inputs): - tracer = framework._dygraph_tracer() - block = framework.default_main_program().current_block() - ivar_inputs = [x._ivar for x in inputs] - - if not hasattr(cls, 'forward_id'): - cls.forward_id = core.PyLayer.num_funcs() + 1 - PyLayer.register_func(cls.forward_id, cls._do_forward) - cls.backward_id = core.PyLayer.num_funcs() + 1 - PyLayer.register_func(cls.backward_id, cls._do_backward) - - iop = core.OpBase(cls.__class__.__name__ + str(cls.forward_id)) - iop.forward_id = cls.forward_id - iop.backward_id = cls.backward_id - block.ops.append(iop) - ivars = tracer.py_trace(iop, ivar_inputs, False) - ret = [] - for ivar in ivars: - tensor = ivar.value().get_tensor() - py_var = framework.Variable( - block, - type=core.VarDesc.VarType.LOD_TENSOR, - name=None, - shape=tensor.shape(), - dtype=tensor._dtype(), - ivar=ivar) - ret.append(py_var) - return ret diff --git a/python/paddle/fluid/tests/unittests/test_imperative_basic.py b/python/paddle/fluid/tests/unittests/test_imperative_basic.py index e7c1f9fda28..f6d3777405c 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_basic.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_basic.py @@ -34,20 +34,6 @@ class MyLayer(fluid.Layer): return [x] -class MyPyLayer(fluid.PyLayer): - def __init__(self): - super(MyPyLayer, self).__init__() - - @staticmethod - def forward(inputs): - return np.tanh(inputs[0]) - - @staticmethod - def backward(inputs): - inp, out, dout = inputs - return np.array(dout) * (1 - np.square(np.array(out))) - - class MLP(fluid.Layer): def __init__(self, name_scope): super(MLP, self).__init__(name_scope) @@ -224,75 +210,6 @@ class TestImperative(unittest.TestCase): l = fluid.Layer("l") self.assertRaises(NotImplementedError, l.forward, []) - def test_pylayer_func_id(self): - - with fluid.dygraph.guard(): - - class PyLayer1(fluid.PyLayer): - def __init__(self): - super(PyLayer1, self).__init__() - - @staticmethod - def forward(input): - return input - - @staticmethod - def backward(input): - return input - - class PyLayer2(fluid.PyLayer): - def __init__(self): - super(PyLayer2, self).__init__() - - @staticmethod - def forward(input): - return input - - @staticmethod - def backward(input): - return input - - py_layer_1 = PyLayer1() - py_layer_2 = PyLayer2() - py_layer_1(fluid.dygraph.base.to_variable(np.ones([2, 2]))) - py_layer_2(fluid.dygraph.base.to_variable(np.ones([2, 2]))) - id = py_layer_1.forward_id - self.assertGreater(id, 0) - self.assertEqual(py_layer_1.backward_id, id + 1) - self.assertEqual(py_layer_2.forward_id, id + 2) - self.assertEqual(py_layer_2.backward_id, id + 3) - py_layer_1(fluid.dygraph.base.to_variable(np.ones([2, 2]))) - self.assertEqual(py_layer_1.forward_id, id) - - def test_pylayer(self): - np_inp = np.ones([2, 2], np.float32) - with fluid.dygraph.guard(): - my_py_layer = MyPyLayer() - var_inp = fluid.dygraph.base.to_variable(np_inp) - outs = my_py_layer(var_inp) - dy_out = np.sum(outs[0].numpy()) - outs[0].backward() - dy_grad = var_inp.gradient() - - with new_program_scope(): - inp = fluid.layers.data( - name="inp", shape=[2, 2], append_batch_size=False) - # TODO(panyx0718): Paddle doesn't diff against data `inp`. - x1 = inp * 1 - # TODO(panyx0718): If reduce_sum is skipped, the result is wrong. - x = fluid.layers.reduce_sum(fluid.layers.tanh(x1)) - param_grads = fluid.backward.append_backward( - x, parameter_list=[x1.name])[0] - exe = fluid.Executor(fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) - - static_out, static_grad = exe.run( - feed={inp.name: np_inp}, - fetch_list=[x.name, param_grads[1].name]) - - self.assertTrue(np.allclose(dy_out, static_out)) - self.assertTrue(np.allclose(dy_grad, static_grad)) - def test_layer_in_out(self): np_inp = np.array([1.0, 2.0, -1.0], dtype=np.float32) with fluid.dygraph.guard(): -- GitLab