clean code of py_layer in dygraph mode,test=develop (#17661)

paddle/fluid/imperative/layer.cc

@@ -33,11 +33,6 @@
 namespace paddle {
 namespace imperative {
 
-const char* PyLayer::kFwdInp = "X";
-const char* PyLayer::kFwdOut = "Out";
-
-std::map<int, py::object> py_funcs_;
-
 using framework::Variable;
 
 namespace detail {
@@ -255,94 +250,85 @@ framework::LoDTensor& VarBase::GradValue() {
 std::map<std::string, std::vector<VarBase*>> OpBase::ApplyGrad(
     BackwardSumMap* bck_map, GradientRef* grad_ref,
     const detail::BackwardStrategy& bck_stratedy) {
-  PADDLE_ENFORCE(!grad_op_descs_.empty() || backward_id_ > 0,
-                 "%s has no backward implementation", Type());
+  PADDLE_ENFORCE(!grad_op_descs_.empty(), "%s has no backward implementation",
+                 Type());
   VLOG(3) << "apply op grad: " << Type();
   std::vector<VarBasePtrMap> tmp_grad_outputs;
-  if (backward_id_ > 0) {
-    VLOG(3) << "py_layer_grad";
-    tmp_grad_outputs.resize(1);
-    tmp_grad_outputs[0][framework::GradVarName(PyLayer::kFwdOut)] =
-        PyLayer::ApplyGrad(
-            backward_id_,
-            grad_input_vars_[0][framework::GradVarName(PyLayer::kFwdInp)]);
-  } else {
-    const size_t grad_op_count = grad_op_descs_.size();
-
-    tmp_grad_outputs.resize(grad_op_count);
-    for (size_t k = 0; k < grad_op_count; ++k) {
-      framework::OpDesc* grad_op_desc = grad_op_descs_[k];
-      platform::RecordEvent record_event(grad_op_desc->Type());
-      auto& grad_output_variable_map = grad_output_vars_[k];
-      VLOG(3) << "apply grad op " << grad_op_desc->Type();
-
-      // Allocate tmp grad output variable
-      for (const auto& it : grad_output_variable_map) {
-        auto& outputs = tmp_grad_outputs[k][it.first];
-        outputs.reserve(it.second.size());
-        for (size_t i = 0; i < it.second.size(); ++i) {
-          VarBase* origin_grad_var_base = it.second[i];
-
-          // Allocate a new variable
-          VarBase* tmp_grad_var_base = new VarBase(
-              string::Sprintf("%s@IGrad", origin_grad_var_base->Name()),
-              origin_grad_var_base->DataType(), origin_grad_var_base->Dims(),
-              place_, true, false);
-          outputs.emplace_back(tmp_grad_var_base);
-        }
+  const size_t grad_op_count = grad_op_descs_.size();
+
+  tmp_grad_outputs.resize(grad_op_count);
+  for (size_t k = 0; k < grad_op_count; ++k) {
+    framework::OpDesc* grad_op_desc = grad_op_descs_[k];
+    platform::RecordEvent record_event(grad_op_desc->Type());
+    auto& grad_output_variable_map = grad_output_vars_[k];
+    VLOG(3) << "apply grad op " << grad_op_desc->Type();
+
+    // Allocate tmp grad output variable
+    for (const auto& it : grad_output_variable_map) {
+      auto& outputs = tmp_grad_outputs[k][it.first];
+      outputs.reserve(it.second.size());
+      for (size_t i = 0; i < it.second.size(); ++i) {
+        VarBase* origin_grad_var_base = it.second[i];
+
+        // Allocate a new variable
+        VarBase* tmp_grad_var_base = new VarBase(
+            string::Sprintf("%s@IGrad", origin_grad_var_base->Name()),
+            origin_grad_var_base->DataType(), origin_grad_var_base->Dims(),
+            place_, true, false);
+        outputs.emplace_back(tmp_grad_var_base);
       }
+    }
 
-      // No need to do compile time infer shape here.
-      // grad_op_desc_->InferShape(*block_);
-      // grad_op_desc->InferVarType(block_);
+    // No need to do compile time infer shape here.
+    // grad_op_desc_->InferShape(*block_);
+    // grad_op_desc->InferVarType(block_);
 
-      std::unique_ptr<framework::OperatorBase> opbase =
-          framework::OpRegistry::CreateOp(*grad_op_desc);
+    std::unique_ptr<framework::OperatorBase> opbase =
+        framework::OpRegistry::CreateOp(*grad_op_desc);
 
-      auto& info = framework::OpInfoMap::Instance().Get(grad_op_desc->Type());
-      if (info.infer_var_type_) {
-        RuntimeInferVarTypeContext infer_var_type_ctx(
-            &grad_input_vars_[k], &tmp_grad_outputs[k], &attrs_);
-        info.infer_var_type_(&infer_var_type_ctx);
-      }
+    auto& info = framework::OpInfoMap::Instance().Get(grad_op_desc->Type());
+    if (info.infer_var_type_) {
+      RuntimeInferVarTypeContext infer_var_type_ctx(
+          &grad_input_vars_[k], &tmp_grad_outputs[k], &attrs_);
+      info.infer_var_type_(&infer_var_type_ctx);
+    }
 
-      framework::OperatorWithKernel* op_kernel =
-          dynamic_cast<framework::OperatorWithKernel*>(opbase.get());
-      PADDLE_ENFORCE_NOT_NULL(op_kernel, "only support op with kernel");
+    framework::OperatorWithKernel* op_kernel =
+        dynamic_cast<framework::OperatorWithKernel*>(opbase.get());
+    PADDLE_ENFORCE_NOT_NULL(op_kernel, "only support op with kernel");
 
-      // Run grad op
-      framework::VariableValueMap grad_invars_map;
-      framework::VariableValueMap grad_outvars_map;
+    // Run grad op
+    framework::VariableValueMap grad_invars_map;
+    framework::VariableValueMap grad_outvars_map;
 
-      for (const auto& it : grad_input_vars_[k]) {
-        auto& grad_invars = grad_invars_map[it.first];
-        grad_invars.reserve(it.second.size());
-        for (const VarBase* grad_inp : it.second) {
-          PADDLE_ENFORCE_NOT_NULL(grad_inp->var_, "op %s input %s nullptr",
-                                  grad_op_desc->Type(), grad_inp->Name());
+    for (const auto& it : grad_input_vars_[k]) {
+      auto& grad_invars = grad_invars_map[it.first];
+      grad_invars.reserve(it.second.size());
+      for (const VarBase* grad_inp : it.second) {
+        PADDLE_ENFORCE_NOT_NULL(grad_inp->var_, "op %s input %s nullptr",
+                                grad_op_desc->Type(), grad_inp->Name());
 
-          grad_invars.emplace_back(grad_inp->var_.get());
-        }
+        grad_invars.emplace_back(grad_inp->var_.get());
       }
+    }
 
-      for (const auto& it : tmp_grad_outputs[k]) {
-        auto& grad_outvars = grad_outvars_map[it.first];
-        grad_outvars.reserve(it.second.size());
-        for (VarBase* grad_out : it.second) {
-          PADDLE_ENFORCE_NOT_NULL(grad_out->var_, "op %s output %s nullptr",
-                                  grad_op_desc->Type(), grad_out->Name());
+    for (const auto& it : tmp_grad_outputs[k]) {
+      auto& grad_outvars = grad_outvars_map[it.first];
+      grad_outvars.reserve(it.second.size());
+      for (VarBase* grad_out : it.second) {
+        PADDLE_ENFORCE_NOT_NULL(grad_out->var_, "op %s output %s nullptr",
+                                grad_op_desc->Type(), grad_out->Name());
 
-          grad_outvars.emplace_back(grad_out->var_.get());
-        }
+        grad_outvars.emplace_back(grad_out->var_.get());
       }
-
-      framework::RuntimeContext ctx(grad_invars_map, grad_outvars_map);
-      framework::Scope scope;
-      PreparedOp p = PreparedOp::Prepare(ctx, *op_kernel, place_);
-      p.op.RuntimeInferShape(scope, place_, ctx);
-      p.func(
-          framework::ExecutionContext(p.op, scope, *p.dev_ctx, p.ctx, nullptr));
     }
+
+    framework::RuntimeContext ctx(grad_invars_map, grad_outvars_map);
+    framework::Scope scope;
+    PreparedOp p = PreparedOp::Prepare(ctx, *op_kernel, place_);
+    p.op.RuntimeInferShape(scope, place_, ctx);
+    p.func(
+        framework::ExecutionContext(p.op, scope, *p.dev_ctx, p.ctx, nullptr));
   }
 
   platform::RecordEvent record_event("merge_grads");
@@ -439,69 +425,5 @@ void VarBase::RunBackward(const detail::BackwardStrategy& bck_stratedy) {
   Autograd().RunBackward(this, bck_stratedy);
 }
 
-void PyLayer::RegisterFunc(int func_id, const py::object& py_func) {
-  py_funcs_[func_id] = py_func;
-}
-
-int PyLayer::NumFuncs() { return py_funcs_.size(); }
-
-std::vector<std::unique_ptr<framework::Variable>> PyLayer::Apply(
-    int func_id, const std::vector<VarBase*>& inputs) {
-  PADDLE_ENFORCE(py_funcs_.find(func_id) != py_funcs_.end());
-  return CallPythonFunc(py_funcs_[func_id], inputs);
-}
-
-std::vector<VarBase*> PyLayer::ApplyGrad(int func_id,
-                                         const std::vector<VarBase*>& inputs) {
-  PADDLE_ENFORCE(py_funcs_.find(func_id) != py_funcs_.end());
-  auto rets = CallPythonFunc(py_funcs_[func_id], inputs);
-
-  std::vector<VarBase*> outs;
-  outs.reserve(rets.size());
-  for (size_t i = 0U; i != rets.size(); ++i) {
-    outs.emplace_back(new VarBase(
-        string::Sprintf("%s_out_%d", framework::GradVarName(PyLayer::kFwdOut),
-                        i),
-        std::move(rets[i]), nullptr, true));
-  }
-
-  return outs;
-}
-
-std::vector<std::unique_ptr<framework::Variable>> PyLayer::CallPythonFunc(
-    const py::object& callable, const std::vector<VarBase*>& ins) {
-  py::gil_scoped_acquire guard;
-  py::tuple in_args(ins.size());
-  for (size_t i = 0; i < ins.size(); ++i) {
-    const framework::LoDTensor& t = ins[i]->var_->Get<framework::LoDTensor>();
-    in_args[i] = t.IsInitialized() ? py::cast(t) : py::cast(nullptr);
-  }
-  VLOG(3) << "pyfunc in " << py::len(in_args);
-
-  // TODO(panyx0718): Who owns the returned LoDTensor.
-  auto ret = callable(in_args);
-  auto ret_tuple = py::cast<py::tuple>(ret);
-  size_t ret_num = py::len(ret_tuple);
-  std::vector<std::unique_ptr<framework::Variable>> outs;
-  outs.reserve(ret_num);
-  VLOG(3) << "pyfunc out " << ret_num;
-  for (size_t i = 0; i < ret_num; ++i) {
-    try {
-      auto* py_out_tensor = py::cast<framework::LoDTensor*>(ret_tuple[i]);
-      PADDLE_ENFORCE_NOT_NULL(py_out_tensor,
-                              "Output tensor %d should not be nullptr", i);
-      auto var =
-          std::unique_ptr<framework::Variable>(new framework::Variable());
-      auto* tensor = var->GetMutable<framework::LoDTensor>();
-      tensor->ShareDataWith(*py_out_tensor);
-      tensor->set_lod(py_out_tensor->lod());
-      outs.emplace_back(std::move(var));
-    } catch (py::cast_error&) {
-      PADDLE_THROW("The %d-th output must be LoDTensor", i);
-    }
-  }
-  return outs;
-}
-
 }  // namespace imperative
 }  // namespace paddle

paddle/fluid/imperative/layer.h

@@ -279,8 +279,6 @@ class PYBIND11_HIDDEN OpBase {
   OpBase(const std::string& type)
       : type_(type),
         trace_id_(-1),
-        forward_id_(-1),
-        backward_id_(-1),
         place_(platform::CPUPlace()),
         backward_hooks_() {}
 
@@ -335,16 +333,10 @@ class PYBIND11_HIDDEN OpBase {
   }
 
   std::string type_;
-  // One of `trace_id_` or `forward_id_` is set, not both.
-  // For pure python PyLayer, use `forward_id_`, otherwise, use trace_id_.
   int trace_id_;
-  int forward_id_;
 
-  // When has backward, one of `grad_op_descs_` or `backward_id_` is set,
-  // not both.
   // Note: each fwd op corresponds to a vector of bwd ops.
   std::vector<framework::OpDesc*> grad_op_descs_;
-  int backward_id_;
 
   platform::Place place_;
 
@@ -373,28 +365,6 @@ class Layer {
   }
 };
 
-class PyLayer {
- public:
-  virtual ~PyLayer() {}
-
-  static const char* kFwdInp;
-  static const char* kFwdOut;
-
-  static void RegisterFunc(int func_id, const py::object& py_func);
-
-  static int NumFuncs();
-
-  static std::vector<std::unique_ptr<framework::Variable>> Apply(
-      int func_id, const std::vector<VarBase*>& inputs);
-
-  static std::vector<VarBase*> ApplyGrad(int func_id,
-                                         const std::vector<VarBase*>& inputs);
-
- private:
-  static std::vector<std::unique_ptr<framework::Variable>> CallPythonFunc(
-      const py::object& callable, const std::vector<VarBase*>& ins);
-};
-
 // infer var type context for imperative mode
 class PYBIND11_HIDDEN RuntimeInferVarTypeContext
     : public framework::InferVarTypeContext {

paddle/fluid/imperative/tracer.cc

@@ -290,56 +290,5 @@ std::set<std::string> Tracer::Trace(OpBase* op, const VarBasePtrMap& inputs,
   return vars_saved_for_backward;
 }
 
-std::vector<VarBase*> Tracer::PyTrace(OpBase* op,
-                                      const std::vector<VarBase*>& inputs,
-                                      bool stop_gradient) {
-  VLOG(3) << "py_trace " << op->Type();
-
-  op->input_vars_[PyLayer::kFwdInp] = inputs;
-
-  std::vector<std::unique_ptr<framework::Variable>> ret_vars =
-      PyLayer::Apply(op->forward_id_, inputs);
-  op->TrackPreOp(PyLayer::kFwdInp, inputs);
-
-  std::vector<VarBase*>& outputs = op->output_vars_[PyLayer::kFwdOut];
-  outputs.reserve(ret_vars.size());
-  for (size_t i = 0U; i != ret_vars.size(); ++i) {
-    VarBase* out = new VarBase(string::Sprintf("%s_out_%d", op->Type(), i),
-                               std::move(ret_vars[i]), nullptr, stop_gradient);
-    outputs.emplace_back(out);
-    out->TrackPreOp(op, PyLayer::kFwdOut, i, stop_gradient);
-  }
-
-  if (!stop_gradient) {
-    VLOG(5) << "start construct backward op";
-    op->grad_input_vars_.resize(1);
-    op->grad_output_vars_.resize(1);
-    auto& grad_input_vars =
-        op->grad_input_vars_[0][framework::GradVarName(PyLayer::kFwdInp)];
-    auto& grad_output_vars =
-        op->grad_output_vars_[0][framework::GradVarName(PyLayer::kFwdOut)];
-
-    for (VarBase* inp : inputs) {
-      grad_input_vars.push_back(inp);
-    }
-    for (VarBase* out : outputs) {
-      grad_input_vars.push_back(out);
-    }
-
-    // TODO(minqiyang): Add GPU support for PyLayer, only support CPU now
-    platform::CPUPlace place;
-    for (VarBase* out : outputs) {
-      InitGrad(out, platform::DeviceContextPool::Instance().Get(place));
-      grad_input_vars.push_back(out->grads_);
-    }
-
-    for (VarBase* inp : inputs) {
-      InitGrad(inp, platform::DeviceContextPool::Instance().Get(place));
-      grad_output_vars.push_back(inp->grads_);
-    }
-  }
-  return outputs;
-}
-
 }  // namespace imperative
 }  // namespace paddle

paddle/fluid/imperative/tracer.h

@@ -53,9 +53,6 @@ class Tracer {
                               const platform::Place expected_place,
                               const bool stop_gradient = false);
 
-  std::vector<VarBase*> PyTrace(OpBase* op, const std::vector<VarBase*>& inputs,
-                                bool stop_gradient = false);
-
  private:
   platform::Place GetPlace(const VarBasePtrMap& inputs);
 

paddle/fluid/pybind/imperative.cc

@@ -18,8 +18,11 @@ limitations under the License. */
 #include <pybind11/complex.h>
 #include <pybind11/functional.h>
 #include <pybind11/stl.h>
+#include <memory>
 
 #include "paddle/fluid/framework/block_desc.h"
+#include "paddle/fluid/imperative/layer.h"
+#include "paddle/fluid/imperative/profiler.h"
 #include "paddle/fluid/imperative/tracer.h"
 #include "paddle/fluid/imperative/type_defs.h"
 
@@ -28,77 +31,182 @@ limitations under the License. */
 namespace paddle {
 namespace pybind {
 
+class Layer : public imperative::Layer {
+ public:
+  using imperative::Layer::Layer;  // Inherit constructors
+
+  std::vector<imperative::VarBase *> Forward(
+      const std::vector<imperative::VarBase *> &inputs) override {
+    PYBIND11_OVERLOAD(std::vector<imperative::VarBase *>, Layer, Forward,
+                      inputs);  // NOLINT
+  }
+};
+
+class PYBIND11_HIDDEN PyOpBase : public imperative::OpBase {
+ public:
+  using imperative::OpBase::OpBase;  // Inherit constructors
+
+  PyOpBase(const std::string &name) : OpBase(name) {}
+};
+
 // Bind Methods
-void BindImperative(pybind11::module* m) {
-  pybind11::class_<imperative::Tracer>(*m, "Tracer", "")
+void BindImperative(pybind11::module *m_ptr) {
+  namespace py = ::pybind11;
+
+  auto &m = *m_ptr;
+
+  py::class_<imperative::detail::BackwardStrategy> backward_strategy(
+      m, "BackwardStrategy", R"DOC()DOC");
+  backward_strategy.def(py::init())
+      .def_property("sort_sum_gradient",
+                    [](const imperative::detail::BackwardStrategy &self) {
+                      return self.sorted_sum_gradient_;
+                    },
+                    [](imperative::detail::BackwardStrategy &self,
+                       bool sorted_sum_gradient) {
+                      self.sorted_sum_gradient_ = sorted_sum_gradient;
+                    });
+
+  m.def("start_imperative_gperf_profiler",
+        []() { imperative::StartProfile(); });
+
+  m.def("stop_imperative_gperf_profiler", []() { imperative::StopProfile(); });
+
+  py::class_<imperative::VarBase>(m, "VarBase", R"DOC()DOC")
+      .def(
+          py::init<const std::string &, paddle::framework::proto::VarType::Type,
+                   const std::vector<int64_t>, const paddle::platform::CPUPlace,
+                   bool, bool>())
+      .def(
+          py::init<const std::string &, paddle::framework::proto::VarType::Type,
+                   const std::vector<int64_t>,
+                   const paddle::platform::CUDAPlace, bool, bool>())
+      .def("_run_backward",
+           [](imperative::VarBase &self,
+              const imperative::detail::BackwardStrategy &bckst) {
+             self.RunBackward(bckst);
+           })
+      .def("_grad_name", &imperative::VarBase::GradName)
+      .def("_grad_value", &imperative::VarBase::GradValue)
+      .def("_clear_gradient", &imperative::VarBase::ClearGradient)
+      .def("_grad_ivar",
+           [](const imperative::VarBase &self) { return self.grads_; },
+           py::return_value_policy::reference)
+      .def("_copy_to",
+           [](const imperative::VarBase &self, const platform::CPUPlace &place,
+              bool blocking) {
+             return self.NewVarBase(place, blocking).release();
+           },
+           py::return_value_policy::take_ownership)
+      .def("_copy_to",
+           [](const imperative::VarBase &self, const platform::CUDAPlace &place,
+              bool blocking) {
+             return self.NewVarBase(place, blocking).release();
+           },
+           py::return_value_policy::take_ownership)
+      .def("value",
+           [](const imperative::VarBase &self) { return self.var_.get(); },
+           py::return_value_policy::reference)
+      .def_property("name", &imperative::VarBase::Name,
+                    &imperative::VarBase::SetName)
+      .def_property_readonly("shape", &imperative::VarBase::Shape)
+      .def_property_readonly("dtype", &imperative::VarBase::DataType)
+      .def_property("persistable", &imperative::VarBase::IsPersistable,
+                    &imperative::VarBase::SetPersistable)
+      .def_property("stop_gradient", &imperative::VarBase::IsStopGradient,
+                    &imperative::VarBase::SetStopGradient);
+
+  py::class_<imperative::OpBase, PyOpBase>(m, "OpBase", R"DOC()DOC")
+      .def(py::init<const std::string &>())
+      .def("register_backward_hooks",
+           [](imperative::OpBase &self, const py::object &callable) {
+             self.RegisterBackwardHooks(callable);
+           })
+      .def_property("_trace_id",
+                    [](const imperative::OpBase &self) {
+                      py::gil_scoped_release release;
+                      return self.trace_id_;
+                    },
+                    [](imperative::OpBase &self, int trace_id) {
+                      py::gil_scoped_release release;
+                      self.trace_id_ = trace_id;
+                    },
+                    py::return_value_policy::reference)
+      .def_property_readonly("type", &imperative::OpBase::Type);
+
+  py::class_<imperative::Layer, Layer /* <--- trampoline*/> layer(m, "Layer");
+  layer.def(py::init<>())
+      .def("forward", [](imperative::Layer &self,
+                         const std::vector<imperative::VarBase *> &inputs) {
+        return self.Forward(inputs);
+      });
+
+  py::class_<imperative::Tracer>(*m, "Tracer", "")
       .def("__init__",
-           [](imperative::Tracer& self, framework::BlockDesc* root_block) {
+           [](imperative::Tracer &self, framework::BlockDesc *root_block) {
              new (&self) imperative::Tracer(root_block);
            })
       .def("trace",
-           [](imperative::Tracer& self, imperative::OpBase* op,
-              const imperative::VarBasePtrMap& inputs,
-              imperative::VarBasePtrMap* outputs,
+           [](imperative::Tracer &self, imperative::OpBase *op,
+              const imperative::VarBasePtrMap &inputs,
+              imperative::VarBasePtrMap *outputs,
               framework::AttributeMap attrs_map,
               const platform::CPUPlace expected_place,
               const bool stop_gradient = false) {
-             pybind11::gil_scoped_release release;
-             return self.Trace(op, inputs, outputs, attrs_map, expected_place,
-                               stop_gradient);
-           })
-      .def("trace",
-           [](imperative::Tracer& self, imperative::OpBase* op,
-              const imperative::VarBasePtrMap& inputs,
-              imperative::VarBasePtrMap* outputs,
-              framework::AttributeMap attrs_map,
-              const platform::CUDAPlace expected_place,
-              const bool stop_gradient = false) {
-             pybind11::gil_scoped_release release;
+             py::gil_scoped_release release;
              return self.Trace(op, inputs, outputs, attrs_map, expected_place,
                                stop_gradient);
            })
-      .def("py_trace", &imperative::Tracer::PyTrace,
-           pybind11::return_value_policy::take_ownership);
+      .def("trace", [](imperative::Tracer &self, imperative::OpBase *op,
+                       const imperative::VarBasePtrMap &inputs,
+                       imperative::VarBasePtrMap *outputs,
+                       framework::AttributeMap attrs_map,
+                       const platform::CUDAPlace expected_place,
+                       const bool stop_gradient = false) {
+        py::gil_scoped_release release;
+        return self.Trace(op, inputs, outputs, attrs_map, expected_place,
+                          stop_gradient);
+      });
 
   // define parallel context
-  pybind11::class_<imperative::ParallelStrategy> parallel_strategy(
-      *m, "ParallelStrategy", "");
-  parallel_strategy.def(pybind11::init())
+  py::class_<imperative::ParallelStrategy> parallel_strategy(
+      m, "ParallelStrategy", "");
+  parallel_strategy.def(py::init())
       .def_property(
           "nranks",
-          [](const imperative::ParallelStrategy& self) { return self.nranks_; },
-          [](imperative::ParallelStrategy& self, int nranks) {
+          [](const imperative::ParallelStrategy &self) { return self.nranks_; },
+          [](imperative::ParallelStrategy &self, int nranks) {
             self.nranks_ = nranks;
           })
       .def_property("local_rank",
-                    [](const imperative::ParallelStrategy& self) {
+                    [](const imperative::ParallelStrategy &self) {
                       return self.local_rank_;
                     },
-                    [](imperative::ParallelStrategy& self, int local_rank) {
+                    [](imperative::ParallelStrategy &self, int local_rank) {
                       self.local_rank_ = local_rank;
                     })
       .def_property(
           "trainer_endpoints",
-          [](const imperative::ParallelStrategy& self) {
+          [](const imperative::ParallelStrategy &self) {
             return self.trainer_endpoints_;
           },
-          [](imperative::ParallelStrategy& self, std::vector<std::string> eps) {
+          [](imperative::ParallelStrategy &self, std::vector<std::string> eps) {
             self.trainer_endpoints_ = eps;
           })
       .def_property("current_endpoint",
-                    [](const imperative::ParallelStrategy& self) {
+                    [](const imperative::ParallelStrategy &self) {
                       return self.current_endpoint_;
                     },
-                    [](imperative::ParallelStrategy& self,
-                       const std::string& ep) { self.current_endpoint_ = ep; });
+                    [](imperative::ParallelStrategy &self,
+                       const std::string &ep) { self.current_endpoint_ = ep; });
 #if defined(PADDLE_WITH_CUDA) && !defined(_WIN32)
-  pybind11::class_<imperative::NCCLParallelContext> nccl_ctx(
-      *m, "NCCLParallelContext");
+  py::class_<imperative::NCCLParallelContext> nccl_ctx(m,
+                                                       "NCCLParallelContext");
 
   nccl_ctx
-      .def(pybind11::init<const imperative::ParallelStrategy&,
-                          const platform::CUDAPlace&>())
-      .def("init", [](imperative::NCCLParallelContext& self) { self.Init(); });
+      .def(py::init<const imperative::ParallelStrategy &,
+                    const platform::CUDAPlace &>())
+      .def("init", [](imperative::NCCLParallelContext &self) { self.Init(); });
 #endif
 }
 

paddle/fluid/pybind/imperative.h

@@ -24,29 +24,6 @@ limitations under the License. */
 namespace paddle {
 namespace pybind {
 
-class Layer : public imperative::Layer {
- public:
-  using imperative::Layer::Layer;  // Inherit constructors
-
-  std::vector<imperative::VarBase*> Forward(
-      const std::vector<imperative::VarBase*>& inputs) override {
-    PYBIND11_OVERLOAD(std::vector<imperative::VarBase*>, Layer, Forward,
-                      inputs);  // NOLINT
-  }
-};
-
-class PYBIND11_HIDDEN PyOpBase : public imperative::OpBase {
- public:
-  using imperative::OpBase::OpBase;  // Inherit constructors
-
-  PyOpBase(const std::string& name) : OpBase(name) {}
-};
-
-class PyVarBase : public imperative::VarBase {
- public:
-  using imperative::VarBase::VarBase;  // Inherit constructors
-};
-
 void BindImperative(pybind11::module* m);
 
 }  // namespace pybind

paddle/fluid/pybind/pybind.cc

@@ -39,8 +39,6 @@ limitations under the License. */
 #include "paddle/fluid/framework/scope_pool.h"
 #include "paddle/fluid/framework/selected_rows.h"
 #include "paddle/fluid/framework/version.h"
-#include "paddle/fluid/imperative/layer.h"
-#include "paddle/fluid/imperative/profiler.h"
 #include "paddle/fluid/memory/allocation/allocator_strategy.h"
 #include "paddle/fluid/memory/allocation/legacy_allocator.h"
 #include "paddle/fluid/operators/activation_op.h"
@@ -190,134 +188,6 @@ PYBIND11_MODULE(core, m) {
   m.def("print_mem_usage",
         []() { return memory::allocation::GPUMemMonitor.PrintMemUsage(); });
 
-  py::class_<imperative::detail::BackwardStrategy> backward_strategy(
-      m, "BackwardStrategy", R"DOC()DOC");
-  backward_strategy.def(py::init())
-      .def_property("sort_sum_gradient",
-                    [](const imperative::detail::BackwardStrategy &self) {
-                      return self.sorted_sum_gradient_;
-                    },
-                    [](imperative::detail::BackwardStrategy &self,
-                       bool sorted_sum_gradient) {
-                      self.sorted_sum_gradient_ = sorted_sum_gradient;
-                    });
-
-  m.def("start_imperative_gperf_profiler",
-        []() { imperative::StartProfile(); });
-
-  m.def("stop_imperative_gperf_profiler", []() { imperative::StopProfile(); });
-
-  py::class_<imperative::VarBase>(m, "VarBase", R"DOC()DOC")
-      .def(
-          py::init<const std::string &, paddle::framework::proto::VarType::Type,
-                   const std::vector<int64_t>, const paddle::platform::CPUPlace,
-                   bool, bool>())
-      .def(
-          py::init<const std::string &, paddle::framework::proto::VarType::Type,
-                   const std::vector<int64_t>,
-                   const paddle::platform::CUDAPlace, bool, bool>())
-      .def("_run_backward",
-           [](imperative::VarBase &self,
-              const imperative::detail::BackwardStrategy &bckst) {
-             self.RunBackward(bckst);
-           })
-      .def("_grad_name", &imperative::VarBase::GradName)
-      .def("_grad_value", &imperative::VarBase::GradValue)
-      .def("_clear_gradient", &imperative::VarBase::ClearGradient)
-      .def("_grad_ivar",
-           [](const imperative::VarBase &self) { return self.grads_; },
-           py::return_value_policy::reference)
-      .def("_copy_to",
-           [](const imperative::VarBase &self, const platform::CPUPlace &place,
-              bool blocking) {
-             std::unique_ptr<imperative::VarBase> new_var =
-                 self.NewVarBase(place, blocking);
-             return new_var.release();
-           },
-           py::return_value_policy::take_ownership)
-      .def("_copy_to",
-           [](const imperative::VarBase &self, const platform::CUDAPlace &place,
-              bool blocking) {
-             std::unique_ptr<imperative::VarBase> new_var =
-                 self.NewVarBase(place, blocking);
-             return new_var.release();
-           },
-           py::return_value_policy::take_ownership)
-      .def("value",
-           [](const imperative::VarBase &self) { return self.var_.get(); },
-           py::return_value_policy::reference)
-      .def_property("name", &imperative::VarBase::Name,
-                    &imperative::VarBase::SetName)
-      .def_property_readonly("shape", &imperative::VarBase::Shape)
-      .def_property_readonly("dtype", &imperative::VarBase::DataType)
-      .def_property("persistable", &imperative::VarBase::IsPersistable,
-                    &imperative::VarBase::SetPersistable)
-      .def_property("stop_gradient", &imperative::VarBase::IsStopGradient,
-                    &imperative::VarBase::SetStopGradient);
-
-  py::class_<imperative::OpBase, PyOpBase>(m, "OpBase", R"DOC()DOC")
-      .def(py::init<const std::string &>())
-      .def("register_backward_hooks",
-           [](imperative::OpBase &self, const py::object &callable) {
-             self.RegisterBackwardHooks(callable);
-           })
-      .def_property("_trace_id",
-                    [](const imperative::OpBase &self) {
-                      pybind11::gil_scoped_release release;
-                      return self.trace_id_;
-                    },
-                    [](imperative::OpBase &self, int trace_id) {
-                      pybind11::gil_scoped_release release;
-                      self.trace_id_ = trace_id;
-                    },
-                    py::return_value_policy::reference)
-      .def_property(
-          "forward_id",
-          [](const imperative::OpBase &self) { return self.forward_id_; },
-          [](imperative::OpBase &self, int forward_id) {
-            self.forward_id_ = forward_id;
-          },
-          py::return_value_policy::reference)
-      .def_property_readonly("type", &imperative::OpBase::Type)
-      .def_property(
-          "backward_id",
-          [](const imperative::OpBase &self) { return self.backward_id_; },
-          [](imperative::OpBase &self, int backward_id) {
-            self.backward_id_ = backward_id;
-          },
-          py::return_value_policy::reference);
-
-  py::class_<imperative::Layer, Layer /* <--- trampoline*/> layer(m, "Layer");
-  layer.def(py::init<>())
-      .def("forward", [](imperative::Layer &self,
-                         const std::vector<imperative::VarBase *> &inputs) {
-        return self.Forward(inputs);
-      });
-
-  py::class_<imperative::PyLayer>(m, "PyLayer")
-      .def(py::init<>())
-      .def_static(
-          "apply",
-          [](int func_id, const std::vector<imperative::VarBase *> &inputs)
-              -> std::vector<imperative::VarBase *> {
-                auto ret_vars = imperative::PyLayer::Apply(func_id, inputs);
-                std::vector<imperative::VarBase *> outputs;
-                outputs.reserve(ret_vars.size());
-                for (size_t i = 0U; i != ret_vars.size(); ++i) {
-                  // TODO(minqiyang): use unique_name generator to set a name
-                  outputs.emplace_back(new imperative::VarBase(
-                      "", std::move(ret_vars[i]), nullptr, true));
-                }
-
-                return outputs;
-              },
-          py::return_value_policy::take_ownership)
-      .def_static("register_func",
-                  [](int func_id, const py::object &callable) {
-                    imperative::PyLayer::RegisterFunc(func_id, callable);
-                  })
-      .def_static("num_funcs", &imperative::PyLayer::NumFuncs);
-
   BindImperative(&m);
 
   py::class_<Tensor>(m, "Tensor", py::buffer_protocol())

python/paddle/fluid/dygraph/layers.py

@@ -25,7 +25,7 @@ from .layer_object_helper import LayerObjectHelper
 from paddle.fluid import framework
 from ..param_attr import ParamAttr
 
-__all__ = ['Layer', 'PyLayer']
+__all__ = ['Layer']
 
 
 class Layer(core.Layer):
@@ -266,76 +266,3 @@ class Layer(core.Layer):
             for layer_name, layer_item in self._sub_layers.items():
                 if layer_item is not None:
                     layer_item.load_dict(stat_dict)
-
-
-class PyLayer(core.PyLayer):
-    """Layers composed of user-defined python codes."""
-
-    def __init__(self):
-        super(PyLayer, self).__init__()
-
-    def train(self):
-        framework._dygraph_tracer().train_mode()
-
-    def eval(self):
-        framework._dygraph_tracer().eval_mode()
-
-    @classmethod
-    def _do_forward(cls, inputs):
-        return cls._to_tuple(cls.forward(inputs))
-
-    @classmethod
-    def _do_backward(cls, inputs):
-        return cls._to_tuple(cls.backward(inputs))
-
-    @staticmethod
-    def _to_tuple(inputs):
-        if not isinstance(inputs, list) and not isinstance(inputs, tuple):
-            inputs = [inputs]
-        ret = []
-        for inp in inputs:
-            if isinstance(inp, core.LoDTensor):
-                ret.append(inp)
-            else:
-                tensor = core.LoDTensor()
-                tensor.set(inp, core.CPUPlace())
-                ret.append(tensor)
-        return tuple(ret)
-
-    @staticmethod
-    def forward(*inputs):
-        raise NotImplementedError
-
-    @staticmethod
-    def backward(*douts):
-        raise NotImplementedError
-
-    @classmethod
-    def __call__(cls, *inputs):
-        tracer = framework._dygraph_tracer()
-        block = framework.default_main_program().current_block()
-        ivar_inputs = [x._ivar for x in inputs]
-
-        if not hasattr(cls, 'forward_id'):
-            cls.forward_id = core.PyLayer.num_funcs() + 1
-            PyLayer.register_func(cls.forward_id, cls._do_forward)
-            cls.backward_id = core.PyLayer.num_funcs() + 1
-            PyLayer.register_func(cls.backward_id, cls._do_backward)
-
-        iop = core.OpBase(cls.__class__.__name__ + str(cls.forward_id))
-        iop.forward_id = cls.forward_id
-        iop.backward_id = cls.backward_id
-        block.ops.append(iop)
-        ivars = tracer.py_trace(iop, ivar_inputs, False)
-        ret = []
-        for ivar in ivars:
-            tensor = ivar.value().get_tensor()
-            py_var = framework.Variable(
-                block,
-                type=core.VarDesc.VarType.LOD_TENSOR,
-                name=None,
-                shape=tensor.shape(),
-                dtype=tensor._dtype(),
-                ivar=ivar)
-            ret.append(py_var)
-        return ret

python/paddle/fluid/tests/unittests/test_imperative_basic.py

@@ -34,20 +34,6 @@ class MyLayer(fluid.Layer):
         return [x]
 
 
-class MyPyLayer(fluid.PyLayer):
-    def __init__(self):
-        super(MyPyLayer, self).__init__()
-
-    @staticmethod
-    def forward(inputs):
-        return np.tanh(inputs[0])
-
-    @staticmethod
-    def backward(inputs):
-        inp, out, dout = inputs
-        return np.array(dout) * (1 - np.square(np.array(out)))
-
-
 class MLP(fluid.Layer):
     def __init__(self, name_scope):
         super(MLP, self).__init__(name_scope)
@@ -224,75 +210,6 @@ class TestImperative(unittest.TestCase):
             l = fluid.Layer("l")
             self.assertRaises(NotImplementedError, l.forward, [])
 
-    def test_pylayer_func_id(self):
-
-        with fluid.dygraph.guard():
-
-            class PyLayer1(fluid.PyLayer):
-                def __init__(self):
-                    super(PyLayer1, self).__init__()
-
-                @staticmethod
-                def forward(input):
-                    return input
-
-                @staticmethod
-                def backward(input):
-                    return input
-
-            class PyLayer2(fluid.PyLayer):
-                def __init__(self):
-                    super(PyLayer2, self).__init__()
-
-                @staticmethod
-                def forward(input):
-                    return input
-
-                @staticmethod
-                def backward(input):
-                    return input
-
-            py_layer_1 = PyLayer1()
-            py_layer_2 = PyLayer2()
-            py_layer_1(fluid.dygraph.base.to_variable(np.ones([2, 2])))
-            py_layer_2(fluid.dygraph.base.to_variable(np.ones([2, 2])))
-            id = py_layer_1.forward_id
-            self.assertGreater(id, 0)
-            self.assertEqual(py_layer_1.backward_id, id + 1)
-            self.assertEqual(py_layer_2.forward_id, id + 2)
-            self.assertEqual(py_layer_2.backward_id, id + 3)
-            py_layer_1(fluid.dygraph.base.to_variable(np.ones([2, 2])))
-            self.assertEqual(py_layer_1.forward_id, id)
-
-    def test_pylayer(self):
-        np_inp = np.ones([2, 2], np.float32)
-        with fluid.dygraph.guard():
-            my_py_layer = MyPyLayer()
-            var_inp = fluid.dygraph.base.to_variable(np_inp)
-            outs = my_py_layer(var_inp)
-            dy_out = np.sum(outs[0].numpy())
-            outs[0].backward()
-            dy_grad = var_inp.gradient()
-
-        with new_program_scope():
-            inp = fluid.layers.data(
-                name="inp", shape=[2, 2], append_batch_size=False)
-            # TODO(panyx0718): Paddle doesn't diff against data `inp`.
-            x1 = inp * 1
-            # TODO(panyx0718): If reduce_sum is skipped, the result is wrong.
-            x = fluid.layers.reduce_sum(fluid.layers.tanh(x1))
-            param_grads = fluid.backward.append_backward(
-                x, parameter_list=[x1.name])[0]
-            exe = fluid.Executor(fluid.CPUPlace(
-            ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0))
-
-            static_out, static_grad = exe.run(
-                feed={inp.name: np_inp},
-                fetch_list=[x.name, param_grads[1].name])
-
-        self.assertTrue(np.allclose(dy_out, static_out))
-        self.assertTrue(np.allclose(dy_grad, static_grad))
-
     def test_layer_in_out(self):
         np_inp = np.array([1.0, 2.0, -1.0], dtype=np.float32)
         with fluid.dygraph.guard():