diff --git a/paddle/fluid/pybind/op_function.h b/paddle/fluid/pybind/op_function.h
index 07e6ff0d87278f1b3e284ee34e49f2fc8939b89a..597ead9327e233df785b58437afce8fa75a058c3 100644
--- a/paddle/fluid/pybind/op_function.h
+++ b/paddle/fluid/pybind/op_function.h
@@ -18,13 +18,45 @@
 #include <pybind11/complex.h>
 #include <pybind11/functional.h>
 #include <pybind11/stl.h>
+#include <memory>
+#include <string>
+#include <vector>
 #include "paddle/fluid/framework/attribute.h"
 #include "paddle/fluid/framework/op_info.h"
 #include "paddle/fluid/framework/variable.h"
 #include "paddle/fluid/imperative/tracer.h"
 #include "paddle/fluid/imperative/type_defs.h"
-
 #include "paddle/fluid/pybind/imperative.h"
 
+namespace py = pybind11;
+namespace paddle {
+namespace pybind {
+static inline void ConstructAttrMapFromPyArgs(framework::AttributeMap* attrs,
+                                              const py::args& args) {
+  PADDLE_ENFORCE_EQ(
+      args.size() % 2, 0,
+      platform::errors::InvalidArgument(
+          "The number of arguments for arributes should be even."));
+  for (size_t i = 0; i < args.size(); i += 2) {
+    auto name = args[i].cast<std::string>();
+    auto value = args[i + 1].cast<framework::Attribute>();
+    (*attrs)[name] = value;
+  }
+}
+
+static inline std::vector<std::shared_ptr<imperative::VarBase>>
+ConstructDuplicableOutput(const size_t num) {
+  auto tracer = imperative::GetCurrentTracer();
+  std::vector<std::shared_ptr<imperative::VarBase>> res;
+  res.reserve(num);
+  for (size_t i = 0; i < num; i++) {
+    auto var_base_name = tracer->GenerateUniqueName();
+    res.emplace_back(new imperative::VarBase(var_base_name));
+  }
+  return res;
+}
+}  // namespace pybind
+}  // namespace paddle
+
 // This include must be the last line
 #include "paddle/fluid/pybind/op_function_impl.h"
diff --git a/paddle/fluid/pybind/op_function_generator.cc b/paddle/fluid/pybind/op_function_generator.cc
index 2c112079ccfc06aae9052680809de41bd1a1e17a..da30a12c1849dc0c8f7a1bdc782c7a1264cda2e7 100644
--- a/paddle/fluid/pybind/op_function_generator.cc
+++ b/paddle/fluid/pybind/op_function_generator.cc
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <algorithm>
 #include <fstream>
 #include <iostream>
 #include <string>
@@ -23,46 +24,93 @@
 #include "paddle/fluid/pybind/pybind.h"
 #include "paddle/fluid/string/string_helper.h"
 
+std::map<std::string, std::set<std::string>> op_ins_map = {
+    {"layer_norm", {"X", "Scale", "Bias"}},
+    {"gru_unit", {"Input", "HiddenPrev", "Weight", "Bias"}},
+    {"label_smooth", {"X", "PriorDist"}},
+    {"assign", {"X"}},
+};
+std::map<std::string, std::set<std::string>> op_passing_out_map = {
+    {"sgd", {"ParamOut"}},
+    {"adam",
+     {"ParamOut", "Moment1Out", "Moment2Out", "Beta1PowOut", "Beta2PowOut"}},
+    {"momentum", {"ParamOut", "VelocityOut"}},
+    {"batch_norm", {"MeanOut", "VarianceOut"}},
+    {"accuracy", {"Correct", "Total"}},
+    {"fill_constant", {"Out"}}};
+
 // clang-format off
 const char* OUT_INITIALIZER_TEMPLATE =
     R"({"%s", {std::shared_ptr<imperative::VarBase>(new imperative::VarBase(tracer->GenerateUniqueName()))}})";
+const char* OUT_DUPLICABLE_INITIALIZER_TEMPLATE = R"({"%s", ConstructDuplicableOutput(%s)})";
+
+const char* INPUT_INITIALIZER_TEMPLATE = R"({"%s", {%s}})";
+const char* INPUT_LIST_INITIALIZER_TEMPLATE = R"({"%s", %s})";
+const char* INPUT_INITIALIZER_TEMPLATE_WITH_NULL = R"(
+    if (%s != nullptr) {
+      ins["%s"] = {%s};
+    }
+)";
+const char* INPUT_INITIALIZER_TEMPLATE_WITH_NULL_LIST = R"(
+    if (%s != nullptr) {
+      ins["%s"] = %s;
+    }
+)";
+
+// if inputs is list, no need {}
+const char* ARG_OUT_NUM = R"(%sNum)";
+const char* ARG_OUT_NUM_TYPE = R"(size_t )";
+
+const char* VAR_TYPE = R"(std::shared_ptr<imperative::VarBase>)";
+const char* VAR_LIST_TYPE = R"(std::vector<std::shared_ptr<imperative::VarBase>>)";
+const char* ARG_TEMPLATE = R"(const %s& %s)";
+
+const char* RETURN_TUPLE_TYPE = R"(std::tuple<%s>)";
+const char* RETURN_TYPE = R"(%s)";
+const char* RETURN_TUPLE_TEMPLATE = R"(std::make_tuple(%s))";
+const char* RETURN_LIST_TEMPLATE = R"(outs["%s"])";
+const char* RETURN_TEMPLATE = R"(outs["%s"][0])";
+
+const char* FUNCTION_ARGS = R"(%s, const py::args& args)";
+const char* FUNCTION_ARGS_NO_INPUT = R"(const py::args& args)";
 
 const char* OP_FUNCTION_TEMPLATE =
 R"(
-inline imperative::NameVarBaseMap %s(const imperative::NameVarBaseMap& ins, const framework::AttributeMap& attrs, 
-  imperative::NameVarBaseMap outs, const std::map<std::string, size_t>& out_nums)
+%s %s(%s)
 {
-  auto tracer = imperative::GetCurrentTracer();
-  if (outs.size() == 0) {
-    if (out_nums.size() == 0) {
-      imperative::NameVarBaseMap outs_ = %s;
-      outs = std::move(outs_);
-    } else {
-      for (auto &pair : out_nums) {
-        for (size_t i = 0; i < pair.second; i ++) {
-          auto var_base_name = tracer->GenerateUniqueName();
-          outs[pair.first].emplace_back(new imperative::VarBase(var_base_name));
-        }
-      }
-    }
-  }
-  
-  tracer->TraceOp("%s", std::move(ins), std::move(outs), std::move(attrs));
-  return outs;
+  framework::AttributeMap attrs;
+  ConstructAttrMapFromPyArgs(&attrs, args);
+  {
+    py::gil_scoped_release release;
+    auto tracer = imperative::GetCurrentTracer();
+    imperative::NameVarBaseMap outs = %s;
+    imperative::NameVarBaseMap ins = %s;
+    %s
+    tracer->TraceOp("%s", ins, outs, attrs);
+    return %s; 
+  }   
 })";
 
-const char* PYBIND_ITEM_TEMPLATE =
-R"(
-  %s.def("%s", &%s, py::arg("ins"), py::arg("attrs")=framework::AttributeMap(), py::arg("outs")=imperative::NameVarBaseMap(), 
-    py::arg("out_nums")=std::map<std::string, size_t>(), py::call_guard<py::gil_scoped_release>());)";
+const char* PYBIND_ITEM_TEMPLATE = R"(  %s.def("%s", &%s);)";
 
 // clang-format on
+static inline bool FindInputInSpecialization(const std::string& op_type,
+                                             const std::string& in_name) {
+  return op_ins_map[op_type].count(in_name);
+}
+
+static inline bool FindOutoutInSpecialization(const std::string& op_type,
+                                              const std::string& out_name) {
+  return op_passing_out_map[op_type].count(out_name);
+}
 
 static std::tuple<std::vector<std::string>, std::vector<std::string>>
 GenerateOpFunctions(const std::string& module_name) {
   auto& op_info_map = paddle::framework::OpInfoMap::Instance().map();
 
   std::vector<std::string> op_function_list, bind_function_list;
+  auto& all_kernels = paddle::framework::OperatorWithKernel::AllOpKernels();
+
   for (auto& pair : op_info_map) {
     auto& op_info = pair.second;
     auto op_proto = op_info.proto_;
@@ -70,27 +118,131 @@ GenerateOpFunctions(const std::string& module_name) {
       continue;
     }
     auto& op_type = op_proto->type();
+    // Skip ooerator which is not inherit form OperatorWithKernel, like while,
+    // since only OperatorWithKernel can run in dygraph mode.
+    if (!all_kernels.count(op_type)) {
+      continue;
+    }
+    std::string input_args = "";
+    std::string ins_initializer = "{";
+    std::string ins_initializer_with_null = "";
+    std::string py_arg = "";
+    for (auto& input : op_proto->inputs()) {
+      auto& in_name = input.name();
+      // skip those dispensable inputs, like ResidualData in conv2d
+      if (input.dispensable() && !FindInputInSpecialization(op_type, in_name)) {
+        continue;
+      }
+      const auto in_type = input.duplicable() ? VAR_LIST_TYPE : VAR_TYPE;
+      auto input_arg = paddle::string::Sprintf(ARG_TEMPLATE, in_type, in_name);
+      input_args += input_arg;
+      input_args += ",";
+
+      if (input.dispensable()) {
+        const auto in_template = input.duplicable()
+                                     ? INPUT_INITIALIZER_TEMPLATE_WITH_NULL_LIST
+                                     : INPUT_INITIALIZER_TEMPLATE_WITH_NULL;
+        ins_initializer_with_null +=
+            paddle::string::Sprintf(in_template, in_name, in_name, in_name);
+      } else {
+        const auto in_template = input.duplicable()
+                                     ? INPUT_LIST_INITIALIZER_TEMPLATE
+                                     : INPUT_INITIALIZER_TEMPLATE;
+        ins_initializer +=
+            paddle::string::Sprintf(in_template, in_name, in_name);
+        ins_initializer += ",";
+      }
+    }
+    if (ins_initializer.back() == ',') {
+      ins_initializer.pop_back();
+    }
+    ins_initializer += "}";
+
+    if (input_args.back() == ',') {
+      input_args.pop_back();
+    }
 
     // Generate outs initializer
     std::string outs_initializer = "{";
+    std::string return_type = "";
+    std::string return_str = "";
 
+    int outs_num = 0;
     for (auto& output : op_proto->outputs()) {
+      if (output.dispensable()) {
+        continue;
+      }
+      const auto out_type = output.duplicable() ? VAR_LIST_TYPE : VAR_TYPE;
+      const auto return_template =
+          output.duplicable() ? RETURN_LIST_TEMPLATE : RETURN_TEMPLATE;
       auto& out_name = output.name();
-      auto out_initializer_str =
-          paddle::string::Sprintf(OUT_INITIALIZER_TEMPLATE, out_name);
+      std::string out_initializer_str;
+      if (FindOutoutInSpecialization(op_type, out_name)) {
+        if (input_args != "") {
+          input_args += ",";
+        }
+        input_args += out_type;
+        input_args += out_name;
+        const auto out_template = output.duplicable()
+                                      ? INPUT_LIST_INITIALIZER_TEMPLATE
+                                      : INPUT_INITIALIZER_TEMPLATE;
+        out_initializer_str +=
+            paddle::string::Sprintf(out_template, out_name, out_name);
+      } else {
+        // There are few Operators that have duplicable output, like `Out` in
+        // split op. We need to specify the number of variables for the
+        // duplicable output, as the argument OutNum;
+        if (output.duplicable()) {
+          if (input_args != "") {
+            input_args += ",";
+          }
+          auto out_num_str = paddle::string::Sprintf(ARG_OUT_NUM, out_name);
+          input_args += ARG_OUT_NUM_TYPE;
+          input_args += out_num_str;
+          out_initializer_str = paddle::string::Sprintf(
+              OUT_DUPLICABLE_INITIALIZER_TEMPLATE, out_name, out_num_str);
+        } else {
+          out_initializer_str =
+              paddle::string::Sprintf(OUT_INITIALIZER_TEMPLATE, out_name);
+        }
+      }
+
+      return_type += out_type;
+      return_type += ",";
+      return_str += paddle::string::Sprintf(return_template, out_name);
+      return_str += ",";
+      outs_num += 1;
+
       outs_initializer += out_initializer_str;
       outs_initializer += ",";
     }
     if (outs_initializer.back() == ',') {
       outs_initializer.pop_back();
+      return_type.pop_back();
+      return_str.pop_back();
     }
     outs_initializer += "}";
+    if (outs_num == 0) {
+      return_type = "void";
+    }
+    if (outs_num > 1) {
+      return_str = paddle::string::Sprintf(RETURN_TUPLE_TEMPLATE, return_str);
+      return_type = paddle::string::Sprintf(RETURN_TUPLE_TYPE, return_type);
+    }
+    std::string function_args = "";
+    if (input_args == "") {
+      function_args =
+          paddle::string::Sprintf(FUNCTION_ARGS_NO_INPUT, input_args);
+    } else {
+      function_args = paddle::string::Sprintf(FUNCTION_ARGS, input_args);
+    }
 
     std::string func_name = "imperative_" + op_type;
-
     // generate op funtcion body
     auto op_function_str = paddle::string::Sprintf(
-        OP_FUNCTION_TEMPLATE, func_name, outs_initializer, op_type);
+        OP_FUNCTION_TEMPLATE, return_type, func_name, function_args,
+        outs_initializer, ins_initializer, ins_initializer_with_null, op_type,
+        return_str);
 
     // generate pybind item
     auto bind_function_str = paddle::string::Sprintf(
@@ -99,7 +251,6 @@ GenerateOpFunctions(const std::string& module_name) {
     op_function_list.emplace_back(std::move(op_function_str));
     bind_function_list.emplace_back(std::move(bind_function_str));
   }
-
   return std::make_tuple(op_function_list, bind_function_list);
 }
 
@@ -119,7 +270,6 @@ int main(int argc, char* argv[]) {
     out << "#include  " + header + "\n";
   }
 
-  // all op functions
   auto op_funcs = GenerateOpFunctions("m");
 
   out << "namespace py = pybind11;"
diff --git a/python/paddle/fluid/dygraph/math_op_patch.py b/python/paddle/fluid/dygraph/math_op_patch.py
index 0424781cf3e4e2c08b7978c77e7f3b63765d8760..dbd8d803f66d9e26978121a46d7b7676ab53c173 100644
--- a/python/paddle/fluid/dygraph/math_op_patch.py
+++ b/python/paddle/fluid/dygraph/math_op_patch.py
@@ -15,7 +15,7 @@
 from __future__ import print_function
 
 from .. import core
-from ..framework import Variable, convert_np_dtype_to_dtype_
+from ..framework import Variable, convert_np_dtype_to_dtype_, _varbase_creator
 from ..layers.layer_function_generator import OpProtoHolder
 from . import to_variable, no_grad
 
@@ -42,17 +42,11 @@ def monkey_patch_math_varbase():
 
     @no_grad
     def create_tensor(value, dtype, shape):
-        value = float(value)
-        inputs = {}
-        attrs = {
-            'dtype': dtype,
-            'shape': shape,
-            'value': value,
-            'force_cpu': False
-        }
-        outs = core.ops.fill_constant(inputs, attrs)
-        outs['Out'][0].stop_gradient = True
-        return outs['Out'][0]
+        out = _varbase_creator(dtype=dtype)
+        out = core.ops.fill_constant(out, 'dtype', dtype, 'shape', shape,
+                                     'value', value, 'force_cpu', False)
+        out.stop_gradient = True
+        return out
 
     def create_scalar(value, dtype):
         return create_tensor(value, dtype, shape=[1])
@@ -102,19 +96,11 @@ def monkey_patch_math_varbase():
                     print("new var's dtype is: {}, numpy dtype is {}".format(new_variable.dtype, new_variable.numpy().dtype))
 
         """
-        inputs = {'X': [self]}
-        attrs = {
-            "in_dtype": self.dtype,
-            "out_dtype": convert_np_dtype_to_dtype_(dtype)
-        }
-        outs = core.ops.cast(inputs, attrs)
-        return outs['Out'][0]
+        return core.ops.cast(self, 'in_dtype', self.dtype, 'out_dtype',
+                             convert_np_dtype_to_dtype_(dtype))
 
     def _scalar_elementwise_op_(var, scale, bias):
-        inputs = {'X': [var]}
-        attrs = {"scale": scale, "bias": bias}
-        outs = core.ops.scale(inputs, attrs)
-        return outs['Out'][0]
+        return core.ops.scale(var, 'scale', scale, 'bias', bias)
 
     def _neg_(var):
         return _scalar_elementwise_op_(var, -1.0, 0.0)
@@ -208,11 +194,8 @@ def monkey_patch_math_varbase():
                 other_var = tmp
 
             axis = -1
-            op = getattr(core.ops, op_type)
-            inputs = {'X': [self], 'Y': [other_var]}
-            attrs = {'axis': axis}
-            outs = op(inputs, attrs)
-            return outs['Out'][0]
+            math_op = getattr(core.ops, op_type)
+            return math_op(self, other_var, 'aixs', axis)
 
         comment = OpProtoHolder.instance().get_op_proto(op_type).comment
 
diff --git a/python/paddle/fluid/dygraph/nn.py b/python/paddle/fluid/dygraph/nn.py
index c9ede3bdefe369785c16730dec9c868eb7c3d9a5..742e29b0ba0fcbe8ea023a48263a8b3f02a57896 100644
--- a/python/paddle/fluid/dygraph/nn.py
+++ b/python/paddle/fluid/dygraph/nn.py
@@ -218,6 +218,17 @@ class Conv2D(layers.Layer):
             is_bias=True)
 
     def forward(self, input):
+        if in_dygraph_mode() and self._l_type == 'conv2d':
+            attrs = ('strides', self._stride, 'paddings', self._padding,
+                     'dilations', self._dilation, 'groups', self._groups
+                     if self._groups else 1, 'use_cudnn', self._use_cudnn)
+            out = core.ops.conv2d(input, self.weight, *attrs)
+            pre_bias = out
+
+            pre_act = dygraph_utils._append_bias_in_dygraph(pre_bias, self.bias,
+                                                            1)
+            return dygraph_utils._append_activation_in_dygraph(pre_act,
+                                                               self._act)
         inputs = {
             'Input': [input],
             'Filter': [self.weight],
@@ -230,17 +241,6 @@ class Conv2D(layers.Layer):
             'use_cudnn': self._use_cudnn,
             'use_mkldnn': False,
         }
-
-        if in_dygraph_mode() and self._l_type == 'conv2d':
-            outs = core.ops.conv2d(inputs, attrs)
-            pre_bias = outs['Output'][0]
-
-            pre_act = dygraph_utils._append_bias_in_dygraph(pre_bias, self.bias,
-                                                            1)
-
-            return dygraph_utils._append_activation_in_dygraph(pre_act,
-                                                               self._act)
-
         pre_bias = self._helper.create_variable_for_type_inference(
             dtype=self._dtype)
 
@@ -825,6 +825,14 @@ class Pool2D(layers.Layer):
         self._l_type = 'pool2d'
 
     def forward(self, input):
+        if in_dygraph_mode():
+            attrs = ('pooling_type', self._pool_type, 'ksize', self._pool_size,
+                     'global_pooling', self._global_pooling, 'strides',
+                     self._pool_stride, 'paddings', self._pool_padding,
+                     'use_cudnn', self._use_cudnn, 'ceil_mode', self._ceil_mode,
+                     'use_mkldnn', False, 'exclusive', self._exclusive)
+            return core.ops.pool2d(input, *attrs)
+
         attrs = {
             "pooling_type": self._pool_type,
             "ksize": self._pool_size,
@@ -838,10 +846,6 @@ class Pool2D(layers.Layer):
         }
         inputs = {"X": [input]}
 
-        if in_dygraph_mode():
-            outs = core.ops.pool2d(inputs, attrs)
-            return outs['Out'][0]
-
         pool_out = self._helper.create_variable_for_type_inference(self._dtype)
 
         self._helper.append_op(
@@ -922,21 +926,20 @@ class Linear(layers.Layer):
             shape=[output_dim], attr=bias_attr, dtype=dtype, is_bias=True)
 
     def forward(self, input):
-        attrs = {
-            "x_num_col_dims": len(input.shape) - 1,
-            "y_num_col_dims": 1,
-        }
-        inputs = {"X": [input], "Y": [self.weight]}
-
         if in_dygraph_mode():
-            outs = core.ops.mul(inputs, attrs)
-            pre_bias = outs['Out'][0]
+            pre_bias = core.ops.mul(input, self.weight, 'x_num_col_dims',
+                                    len(input.shape) - 1, 'y_num_col_dims', 1)
 
             pre_act = dygraph_utils._append_bias_in_dygraph(
                 pre_bias, self.bias, axis=len(input.shape) - 1)
 
             return dygraph_utils._append_activation_in_dygraph(pre_act,
                                                                self._act)
+        attrs = {
+            "x_num_col_dims": len(input.shape) - 1,
+            "y_num_col_dims": 1,
+        }
+        inputs = {"X": [input], "Y": [self.weight]}
 
         tmp = self._helper.create_variable_for_type_inference(self._dtype)
         self._helper.append_op(
@@ -1128,8 +1131,22 @@ class BatchNorm(layers.Layer):
         # mean and mean_out share the same memory
         mean_out = self._mean
         # variance and variance out share the same memory
-
         variance_out = self._variance
+
+        if in_dygraph_mode():
+            _is_test = (not _dygraph_tracer()._train_mode) and (
+                not self._trainable_statistics)
+            attrs = ("momentum", self._momentum, "epsilon", self._epsilon,
+                     "is_test", _is_test, "data_layout", self._data_layout,
+                     "use_mkldnn", False, "fuse_with_relu",
+                     self._fuse_with_relu, "use_global_stats",
+                     self._use_global_stats)
+            batch_norm_out, _, _, _, _ = core.ops.batch_norm(
+                input, self.weight, self.bias, self._mean, self._variance,
+                mean_out, variance_out, *attrs)
+            return dygraph_utils._append_activation_in_dygraph(
+                batch_norm_out, act=self._act)
+
         attrs = {
             "momentum": self._momentum,
             "epsilon": self._epsilon,
@@ -1149,20 +1166,12 @@ class BatchNorm(layers.Layer):
             "Variance": [self._variance]
         }
 
-        if in_dygraph_mode():
-            attrs['is_test'] = not _dygraph_tracer()._train_mode
-            saved_mean = _varbase_creator(dtype=self._dtype)
-            saved_variance = _varbase_creator(dtype=self._dtype)
-            batch_norm_out = _varbase_creator(dtype=self._dtype)
-            batch_norm_out.stop_gradient = False
-            # inplace is not supported currently
-        else:
-            saved_mean = self._helper.create_variable_for_type_inference(
-                dtype=self._dtype, stop_gradient=True)
-            saved_variance = self._helper.create_variable_for_type_inference(
-                dtype=self._dtype, stop_gradient=True)
-            batch_norm_out = input if self._in_place else self._helper.create_variable_for_type_inference(
-                self._dtype)
+        saved_mean = self._helper.create_variable_for_type_inference(
+            dtype=self._dtype, stop_gradient=True)
+        saved_variance = self._helper.create_variable_for_type_inference(
+            dtype=self._dtype, stop_gradient=True)
+        batch_norm_out = input if self._in_place else self._helper.create_variable_for_type_inference(
+            self._dtype)
 
         outputs = {
             "Y": [batch_norm_out],
@@ -1172,11 +1181,6 @@ class BatchNorm(layers.Layer):
             "SavedVariance": [saved_variance]
         }
 
-        if in_dygraph_mode():
-            outs = core.ops.batch_norm(inputs, attrs, outputs)
-            return dygraph_utils._append_activation_in_dygraph(
-                batch_norm_out, act=self._act)
-
         self._helper.append_op(
             type="batch_norm", inputs=inputs, outputs=outputs, attrs=attrs)
 
@@ -1315,6 +1319,12 @@ class Embedding(layers.Layer):
             is_bias=False)
 
     def forward(self, input):
+        if in_dygraph_mode():
+            return core.ops.lookup_table_v2(
+                self.weight, input, 'is_sparse', self._is_sparse,
+                'is_distributed', self._is_distributed, 'remote_prefetch',
+                self._remote_prefetch, 'padding_idx', self._padding_idx)
+
         attrs = {
             'is_sparse': self._is_sparse,
             'is_distributed': self._is_distributed,
@@ -1322,11 +1332,6 @@ class Embedding(layers.Layer):
             'padding_idx': self._padding_idx
         }
 
-        if in_dygraph_mode():
-            inputs = {'Ids': [input], 'W': [self.weight]}
-            outs = core.ops.lookup_table_v2(inputs, attrs)
-            return outs['Out'][0]
-
         out = self._helper.create_variable_for_type_inference(self._dtype)
         self._helper.append_op(
             type='lookup_table_v2',
@@ -1436,6 +1441,7 @@ class LayerNorm(layers.Layer):
         else:
             if self._param_attr:
                 logging.warn("param_attr are only available with scale is True")
+            self.weight = None
 
         if self._shift:
             assert self._bias_attr is not False
@@ -1447,6 +1453,7 @@ class LayerNorm(layers.Layer):
         else:
             if self._bias_attr:
                 logging.warn("bias_attr are only available with shift is True")
+            self.bias = None
 
     def forward(self, input):
         input_shape = list(input.shape)
@@ -1460,24 +1467,25 @@ class LayerNorm(layers.Layer):
                 'Given normalized_shape is ' + str_normalized_shape +
                 ', expected input with shape [*, ' + str_normalized_shape[
                     1:] + ', but got input shape ' + str(input_shape))
+
+        if in_dygraph_mode():
+            pre_act, _, _ = core.ops.layer_norm(
+                input, self.weight, self.bias, 'epsilon', self._epsilon,
+                'begin_norm_axis', self._begin_norm_axis)
+            return dygraph_utils._append_activation_in_dygraph(
+                pre_act, act=self._act)
+
         inputs = dict()
         inputs['X'] = [input]
         if self._scale:
             inputs['Scale'] = [self.weight]
         if self._shift:
             inputs['Bias'] = [self.bias]
-
         attrs = {
             "epsilon": self._epsilon,
             "begin_norm_axis": self._begin_norm_axis
         }
 
-        if in_dygraph_mode():
-            outs = core.ops.layer_norm(inputs, attrs)
-            pre_act = outs['Y'][0]
-            return dygraph_utils._append_activation_in_dygraph(
-                pre_act, act=self._act)
-
         # create output
         mean_out = self._helper.create_variable_for_type_inference(
             dtype=self._dtype, stop_gradient=True)
@@ -1642,6 +1650,12 @@ class GRUUnit(layers.Layer):
             attr=bias_attr, shape=bias_size, dtype=dtype, is_bias=True)
 
     def forward(self, input, hidden):
+        if in_dygraph_mode():
+            gate, reset_hidden_pre, updated_hidden = core.ops.gru_unit(
+                input, hidden, self.weight, self.bias, 'activation',
+                self.activation, 'gate_activation', self.gate_activation)
+            return updated_hidden, reset_hidden_pre, gate
+
         inputs = {
             'Input': [input],
             'HiddenPrev': [hidden],
@@ -1653,12 +1667,6 @@ class GRUUnit(layers.Layer):
             'activation': self.activation,
             'gate_activation': self.gate_activation,
         }
-
-        if in_dygraph_mode():
-            outs = core.ops.gru_unit(inputs, attrs)
-            return outs['Hidden'][0], outs['ResetHiddenPrev'][0], outs['Gate'][
-                0]
-
         gate = self._helper.create_variable_for_type_inference(self._dtype)
         reset_hidden_pre = self._helper.create_variable_for_type_inference(
             self._dtype)
@@ -2310,6 +2318,18 @@ class Conv2DTranspose(layers.Layer):
             is_bias=True)
 
     def forward(self, input):
+        if in_dygraph_mode():
+            op = getattr(core.ops, self._op_type)
+            out = op(input, self.weight, 'output_size', self._output_size,
+                     'strides', self._stride, 'paddings', self._padding,
+                     'dilations', self._dilation, 'groups', self._groups,
+                     'use_cudnn', self._use_cudnn)
+            pre_bias = out
+            pre_act = dygraph_utils._append_bias_in_dygraph(pre_bias, self.bias,
+                                                            1)
+            return dygraph_utils._append_activation_in_dygraph(
+                pre_act, act=self._act)
+
         inputs = {'Input': [input], 'Filter': [self.weight]}
         attrs = {
             'output_size': self._output_size,
@@ -2320,15 +2340,6 @@ class Conv2DTranspose(layers.Layer):
             'use_cudnn': self._use_cudnn
         }
 
-        if in_dygraph_mode():
-            op = getattr(core.ops, self._op_type)
-            outs = op(inputs, attrs)
-            pre_bias = outs['Output'][0]
-            pre_act = dygraph_utils._append_bias_in_dygraph(pre_bias, self.bias,
-                                                            1)
-            return dygraph_utils._append_activation_in_dygraph(
-                pre_act, act=self._act)
-
         pre_bias = self._helper.create_variable_for_type_inference(
             dtype=input.dtype)
         self._helper.append_op(
diff --git a/python/paddle/fluid/dygraph_utils.py b/python/paddle/fluid/dygraph_utils.py
index f8faa3d55b751583046200494e4a1897f4582b75..6ab3c27fa4cfc2351f6c7f4c50190a3f6c24ef01 100644
--- a/python/paddle/fluid/dygraph_utils.py
+++ b/python/paddle/fluid/dygraph_utils.py
@@ -33,15 +33,15 @@ def _append_activation_in_dygraph(input,
     """
     if not act:
         return input
-    attrs = {}
-    if (use_cudnn is not None) and use_cudnn:
-        attrs['use_cudnn'] = use_cudnn
-    if (use_mkldnn is not None) and use_mkldnn:
-        attrs['use_mkldnn'] = use_mkldnn
-    inputs = {"X": [input]}
+
+    attrs = ()
+    if use_cudnn:
+        attrs = ('use_cudnn', use_cudnn)
+    if use_mkldnn:
+        attrs += ('use_mkldnn', use_mkldnn)
+
     act_op = getattr(core.ops, act)
-    res = act_op(inputs, attrs)
-    return res['Out'][0]
+    return act_op(input, *attrs)
 
 
 @dygraph_only
@@ -58,7 +58,4 @@ def _append_bias_in_dygraph(input, bias=None, axis=1):
     if not bias:
         return input
 
-    attrs = {'axis': axis}
-    inputs = {'X': [input], 'Y': [bias]}
-    outs = core.ops.elementwise_add(inputs, attrs)
-    return outs['Out'][0]
+    return core.ops.elementwise_add(input, bias, 'axis', axis)
diff --git a/python/paddle/fluid/layers/layer_function_generator.py b/python/paddle/fluid/layers/layer_function_generator.py
index d113868fbc90d571190f69026da634fefef4f8b2..511f1274db0a13a6f59b70b3549d2072b93bf56b 100755
--- a/python/paddle/fluid/layers/layer_function_generator.py
+++ b/python/paddle/fluid/layers/layer_function_generator.py
@@ -253,10 +253,8 @@ def generate_activation_fn(op_type):
 
     def func(x, name=None):
         if in_dygraph_mode():
-            inputs = {'X': [x]}
             op = getattr(core.ops, op_type)
-            outs = op(inputs)
-            return outs['Out'][0]
+            return op(x)
 
         check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'],
                                  op_type)
diff --git a/python/paddle/fluid/layers/loss.py b/python/paddle/fluid/layers/loss.py
index 23c062a419b4d979ebfb021fb68dac2c74783ccb..5eda55d76a68c5b08a7c7ef4941517ae55a83d04 100644
--- a/python/paddle/fluid/layers/loss.py
+++ b/python/paddle/fluid/layers/loss.py
@@ -238,13 +238,13 @@ def cross_entropy(input, label, soft_label=False, ignore_index=kIgnoreIndex):
     if not soft_label:
         return cross_entropy2(input, label, ignore_index)
 
+    if in_dygraph_mode():
+        return core.ops.cross_entropy(input, label, "soft_label", soft_label,
+                                      "ignore_index", ignore_index)
+
     inputs = {'X': [input], 'Label': [label]}
     attrs = {"soft_label": soft_label, "ignore_index": ignore_index}
 
-    if in_dygraph_mode():
-        outs = core.ops.cross_entropy(inputs, attrs)
-        return outs['Y'][0]
-
     check_variable_and_dtype(input, 'input', ['float16', 'float32', 'float64'],
                              'cross_entropy')
     helper = LayerHelper('cross_entropy', **locals())
@@ -255,13 +255,13 @@ def cross_entropy(input, label, soft_label=False, ignore_index=kIgnoreIndex):
 
 
 def cross_entropy2(input, label, ignore_index=kIgnoreIndex):
-    inputs = {'X': [input], 'Label': [label]}
-    attrs = {'ignore_index': ignore_index}
-
     if in_dygraph_mode():
-        outs = core.ops.cross_entropy2(inputs, attrs)
-        return outs['Y'][0]
+        loss, _, _ = core.ops.cross_entropy2(input, label, 'ignore_index',
+                                             ignore_index)
+        return loss
 
+    inputs = {'X': [input], 'Label': [label]}
+    attrs = {'ignore_index': ignore_index}
     check_variable_and_dtype(input, 'input', ['float16', 'float32', 'float64'],
                              'cross_entropy2')
     helper = LayerHelper('cross_entropy2', **locals())
@@ -1233,21 +1233,22 @@ def softmax_with_cross_entropy(logits,
             out = fluid.layers.softmax_with_cross_entropy(
                 logits=fc, label=label)
     """
+    if in_dygraph_mode():
+        softmax, loss = core.ops.softmax_with_cross_entropy(
+            logits, label, 'soft_label', soft_label, 'ignore_index',
+            ignore_index, 'numeric_stable_mode', numeric_stable_mode, 'axis',
+            axis)
+        if not return_softmax:
+            return loss
+        else:
+            return loss, softmax
+
     attrs = {
         'soft_label': soft_label,
         'ignore_index': ignore_index,
         'numeric_stable_mode': numeric_stable_mode,
         'axis': axis
     }
-
-    if in_dygraph_mode():
-        inputs = {'Logits': [logits], 'Label': [label]}
-        outs = core.ops.softmax_with_cross_entropy(inputs, attrs)
-        if not return_softmax:
-            return outs['Loss'][0]
-        else:
-            return outs['Loss'][0], outs['Softmax'][0]
-
     helper = LayerHelper('softmax_with_cross_entropy', **locals())
     softmax = helper.create_variable_for_type_inference(dtype=logits.dtype)
     loss = helper.create_variable_for_type_inference(dtype=logits.dtype)
diff --git a/python/paddle/fluid/layers/metric_op.py b/python/paddle/fluid/layers/metric_op.py
index de92f3c3cfe24433196b1c8ee55b0e0a8f833a8c..3b6207c2c607435b2806c82ddeded6fe5625eddb 100644
--- a/python/paddle/fluid/layers/metric_op.py
+++ b/python/paddle/fluid/layers/metric_op.py
@@ -74,24 +74,15 @@ def accuracy(input, label, k=1, correct=None, total=None):
             #[array([0.6666667], dtype=float32)]
     """
     if in_dygraph_mode():
-        topk_out, topk_indices = nn.topk(input, k=k)
-        inputs = {
-            "Out": [topk_out],
-            "Indices": [topk_indices],
-            "Label": [label]
-        }
-        acc_out = _varbase_creator(dtype="float32")
         if correct is None:
-            correct = _varbase_creator(dtype="int64")
+            correct = _varbase_creator(dtype="int32")
         if total is None:
-            total = _varbase_creator(dtype="int64")
-        outputs = {
-            "Accuracy": [acc_out],
-            "Correct": [correct],
-            "Total": [total]
-        }
-        outs = core.ops.accuracy(inputs, {}, outputs)
-        return outs['Accuracy'][0]
+            total = _varbase_creator(dtype="int32")
+
+        topk_out, topk_indices = nn.topk(input, k=k)
+        _acc, _, _ = core.ops.accuracy(topk_out, topk_indices, label, correct,
+                                       total)
+        return _acc
 
     helper = LayerHelper("accuracy", **locals())
     check_variable_and_dtype(input, 'input', ['float16', 'float32', 'float64'],
@@ -99,9 +90,9 @@ def accuracy(input, label, k=1, correct=None, total=None):
     topk_out, topk_indices = nn.topk(input, k=k)
     acc_out = helper.create_variable_for_type_inference(dtype="float32")
     if correct is None:
-        correct = helper.create_variable_for_type_inference(dtype="int64")
+        correct = helper.create_variable_for_type_inference(dtype="int32")
     if total is None:
-        total = helper.create_variable_for_type_inference(dtype="int64")
+        total = helper.create_variable_for_type_inference(dtype="int32")
     helper.append_op(
         type="accuracy",
         inputs={
diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py
index f7f1c867b660efe4ce4b57c5a45058f38356683d..063172a149f2ad9d4a246d1839b61fd405edc66c 100644
--- a/python/paddle/fluid/layers/nn.py
+++ b/python/paddle/fluid/layers/nn.py
@@ -192,11 +192,8 @@ def _elementwise_op_in_dygraph(x,
                                act=None,
                                use_mkldnn=False,
                                op_name=None):
-    attrs = {'axis': axis, 'use_mkldnn': use_mkldnn}
-    inputs = {'X': [x], 'Y': [y]}
     op = getattr(core.ops, op_name)
-    outs = op(inputs, attrs)
-    out = outs['Out'][0]
+    out = op(x, y, 'axis', axis, 'use_mkldnn', use_mkldnn)
 
     return dygraph_utils._append_activation_in_dygraph(
         out, act, use_mkldnn=use_mkldnn)
@@ -835,11 +832,16 @@ def dropout(x,
         return attrs
 
     if in_dygraph_mode():
-        attrs = get_attrs(default_main_program(), dropout_prob, is_test, seed)
-        attrs['is_test'] = not _dygraph_tracer()._train_mode
-        inputs = {'X': [x]}
-        outs = core.ops.dropout(inputs, attrs)
-        return outs['Out'][0]
+        if (seed is None or
+                seed == 0) and default_main_program().random_seed != 0:
+            seed = default_main_program().random_seed
+        seed = seed if seed is not None else 0
+        _is_test = not _dygraph_tracer()._train_mode
+        out, mask = core.ops.dropout(x, 'dropout_prob', dropout_prob, 'is_test',
+                                     _is_test, 'fix_seed', seed is not None,
+                                     'seed', seed, 'dropout_implementation',
+                                     dropout_implementation)
+        return out
 
     helper = LayerHelper('dropout', **locals())
     check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'],
@@ -1118,12 +1120,12 @@ def softmax(input, use_cudnn=False, name=None, axis=-1):
                              fetch_list=[result[0]])
             print(output)
     """
-    inputs = {"X": [input]}
-    attrs = {"axis": axis, "use_cudnn": use_cudnn}
 
     if in_dygraph_mode():
-        outs = core.ops.softmax(inputs, attrs)
-        return outs['Out'][0]
+        return core.ops.softmax(input, 'axis', axis, 'use_cudnn', use_cudnn)
+
+    inputs = {"X": [input]}
+    attrs = {"axis": axis, "use_cudnn": use_cudnn}
 
     helper = LayerHelper('softmax', **locals())
     check_variable_and_dtype(input, 'input', ['float16', 'float32', 'float64'],
@@ -3887,17 +3889,17 @@ def reduce_sum(input, dim=None, keep_dim=False, name=None):
     """
     if dim is not None and not isinstance(dim, list):
         dim = [dim]
+
+    if in_dygraph_mode():
+        reduce_all = True if dim == None or dim == [] else False
+        dim = dim if dim != None and dim != [] else [0]
+        return core.ops.reduce_sum(input, 'dim', dim, 'keep_dim', keep_dim,
+                                   'reduce_all', reduce_all)
     attrs = {
         'dim': dim if dim != None and dim != [] else [0],
         'keep_dim': keep_dim,
         'reduce_all': True if dim == None or dim == [] else False
     }
-
-    if in_dygraph_mode():
-        inputs = {'X': [input]}
-        outs = core.ops.reduce_sum(inputs, attrs)
-        return outs['Out'][0]
-
     check_variable_and_dtype(
         input, 'input', ['float32', 'float64', 'int32', 'int64'], 'reduce_sum')
     helper = LayerHelper('reduce_sum', **locals())
@@ -3962,17 +3964,17 @@ def reduce_mean(input, dim=None, keep_dim=False, name=None):
 
     if dim is not None and not isinstance(dim, list):
         dim = [dim]
+
+    if in_dygraph_mode():
+        reduce_all = True if dim == None or dim == [] else False
+        dim = dim if dim != None and dim != [] else [0]
+        return core.ops.reduce_mean(input, 'dim', dim, 'keep_dim', keep_dim,
+                                    'reduce_all', reduce_all)
     attrs = {
         'dim': dim if dim != None and dim != [] else [0],
         'keep_dim': keep_dim,
         'reduce_all': True if dim == None or dim == [] else False
     }
-
-    if in_dygraph_mode():
-        inputs = {'X': [input]}
-        outs = core.ops.reduce_mean(inputs, attrs)
-        return outs['Out'][0]
-
     check_variable_and_dtype(
         input, 'input', ['float32', 'float64', 'int32', 'int64'], 'reduce_mean')
     helper = LayerHelper('reduce_mean', **locals())
@@ -4333,19 +4335,20 @@ def split(input, num_or_sections, dim=-1, name=None):
             # x2.shape [3, 4, 5]
     """
     if in_dygraph_mode():
-        inputs = {'X': [input]}
-        attrs = {}
+        num = None
+        attrs = ()
+
         if isinstance(dim, Variable):
             dim = dim.numpy()
             assert dim.shape == (1,
                                  ), "dim of type Variable should have shape [1]"
             dim = dim[0]
         dim = (len(input.shape) + dim) if dim < 0 else dim
-        attrs['axis'] = dim
+        attrs += ('axis', dim)
 
         if isinstance(num_or_sections, int):
             num = num_or_sections
-            attrs['num'] = num_or_sections
+            attrs += ('num', num_or_sections)
         elif isinstance(num_or_sections, (list, tuple)):
             num = len(num_or_sections)
             if utils._contain_var(num_or_sections):
@@ -4354,14 +4357,12 @@ def split(input, num_or_sections, dim=-1, name=None):
                     "received %s, which contains Variable." %
                     (type(num_or_sections)))
             else:
-                attrs['sections'] = list(num_or_sections)
+                attrs += ('sections', list(num_or_sections))
         else:
             raise TypeError(
                 "The type of 'num_or_sections' in split must be int or list in Dygraph mode, but "
                 "received %s." % (type(num_or_sections)))
-
-        res = core.ops.split(inputs, attrs, {}, {'Out': num})
-        return res['Out']
+        return core.ops.split(input, num, *attrs)
 
     if not isinstance(num_or_sections, (int, list, tuple)):
         raise TypeError(
@@ -4596,9 +4597,8 @@ def matmul(x, y, transpose_x=False, transpose_y=False, alpha=1.0, name=None):
     }
 
     if in_dygraph_mode():
-        inputs = {'X': [x], 'Y': [y]}
-        outs = core.ops.matmul(inputs, attrs)
-        return outs['Out'][0]
+        return core.ops.matmul(x, y, 'transpose_X', transpose_x, 'transpose_Y',
+                               transpose_y, 'alpha', float(alpha))
 
     def __check_input(x, y):
         var_names = {'x': x, 'y': y}
@@ -4716,20 +4716,15 @@ def topk(input, k, name=None):
             vk_values, vk_indices = layers.topk(input2, k=vk) #vk_values.shape=[None, 13, k], vk_indices.shape=[None, 13, k]
 
     """
-    inputs = {"X": [input]}
-    attrs = {}
-
     if in_dygraph_mode():
-        if isinstance(k, Variable):
-            k = k.numpy()
-            assert k.shape == (1, ), "k of type Variable should have shape [1]"
-            k = k[0]
-        attrs = {'k': k}
-        outs = core.ops.top_k(inputs, attrs)
-        outs['Out'][0].stop_gradient = True
-        outs['Indices'][0].stop_gradient = True
-        return outs['Out'][0], outs['Indices'][0]
+        _k = k.numpy().item(0) if isinstance(k, Variable) else k
+        out, indices = core.ops.top_k(input, 'k', _k)
+        out.stop_gradient = True
+        indices.stop_gradient = True
+        return out, indices
 
+    inputs = {"X": [input]}
+    attrs = {}
     if isinstance(k, Variable):
         inputs['K'] = [k]
     else:
@@ -4965,10 +4960,8 @@ def transpose(x, perm, name=None):
 
     """
     if in_dygraph_mode():
-        attrs = {'axis': perm}
-        inputs = {'X': [x]}
-        outs = core.ops.transpose2(inputs, attrs)
-        return outs['Out'][0]
+        out, _ = core.ops.transpose2(x, 'axis', perm)
+        return out
 
     check_variable_and_dtype(
         x, 'x', ['float16', 'float32', 'float64', 'int32', 'int64'],
@@ -5413,11 +5406,10 @@ def one_hot(input, depth, allow_out_of_range=False):
             assert depth.shape == (
                 1, ), "depth of type Variable should have shape [1]"
             depth = depth[0]
-        inputs = {'X': [input]}
-        attrs = {'depth': depth, 'allow_out_of_range': allow_out_of_range}
-        outs = core.ops.one_hot(inputs, attrs)
-        outs['Out'][0].stop_gradient = True
-        return outs['Out'][0]
+        out = core.ops.one_hot(input, 'depth', depth, 'allow_out_of_range',
+                               allow_out_of_range)
+        out.stop_gradient = True
+        return out
 
     helper = LayerHelper("one_hot", **locals())
     one_hot_out = helper.create_variable_for_type_inference(dtype='float32')
@@ -5597,9 +5589,7 @@ def reshape(x, shape, actual_shape=None, act=None, inplace=False, name=None):
                 "The type of 'shape' in reshape must be list[int] or tuple(int) in Dygraph mode, but "
                 "received %s." % type(shape))
 
-        inputs = {'X': [x]}
-        outs = core.ops.reshape2(inputs, attrs)
-        out = outs['Out'][0]
+        out, _ = core.ops.reshape2(x, 'shape', shape)
         return dygraph_utils._append_activation_in_dygraph(out, act)
 
     check_variable_and_dtype(
@@ -6274,12 +6264,8 @@ def label_smooth(label,
         raise ValueError("The value of epsilon must be between 0 and 1.")
 
     if in_dygraph_mode():
-        inputs = {"X": [label]}
-        if prior_dist:
-            inputs["PriorDist"] = [prior_dist]
-        attrs = {"epsilon": float(epsilon)}
-        outs = core.ops.label_smooth(inputs, attrs)
-        return outs['Out'][0]
+        return core.ops.label_smooth(label, prior_dist, 'epsilon',
+                                     float(epsilon))
 
     helper = LayerHelper("label_smooth", **locals())
     label.stop_gradient = True
@@ -7854,11 +7840,10 @@ def log(x, name=None):
             res_val, = exe.run(fluid.default_main_program(), feed={'x':x_i}, fetch_list=[res])
             print(res_val) # [[0.], [0.6931472]]
     """
-    inputs = {'X': [x]}
     if in_dygraph_mode():
-        outs = core.ops.log(inputs)
-        return outs['Out'][0]
+        return core.ops.log(x)
 
+    inputs = {'X': [x]}
     helper = LayerHelper('log', **locals())
     dtype = helper.input_dtype(input_param_name='x')
     out = helper.create_variable_for_type_inference(dtype)
@@ -7894,11 +7879,10 @@ def relu(x, name=None):
                 # [[0.  0. ]
                 #  [1.  2.6]]
 """
-    inputs = {'X': [x]}
     if in_dygraph_mode():
-        outs = core.ops.relu(inputs)
-        return outs['Out'][0]
+        return core.ops.relu(x)
 
+    inputs = {'X': [x]}
     helper = LayerHelper('relu', **locals())
     dtype = helper.input_dtype(input_param_name='x')
     out = helper.create_variable_for_type_inference(dtype)
@@ -8480,6 +8464,13 @@ def pad2d(input,
           result = fluid.layers.pad2d(input=data, paddings=[1, 2, 3, 4],
                                       mode='reflect')
     """
+
+    if in_dygraph_mode():
+        _paddings = paddings.numpy().tolist() if isinstance(
+            paddings, Variable) else paddings
+        return core.ops.pad2d(input, 'mode', mode, 'pad_value', pad_value,
+                              'data_format', data_format, 'paddings', _paddings)
+
     attrs = {'mode': mode, 'pad_value': pad_value, 'data_format': data_format}
     inputs = {'X': [input]}
     if isinstance(paddings, Variable):
@@ -8488,10 +8479,6 @@ def pad2d(input,
     else:
         attrs['paddings'] = paddings
 
-    if in_dygraph_mode():
-        outs = core.ops.pad2d(inputs, attrs)
-        return outs['Out'][0]
-
     helper = LayerHelper('pad2d', **locals())
 
     assert mode in ['reflect', 'edge', 'constant'
@@ -8927,12 +8914,11 @@ def leaky_relu(x, alpha=0.02, name=None):
             res_val, = exe.run(fluid.default_main_program(), feed={'x':x_i}, fetch_list=[res])
             print(res_val) # [[-0.1, 2], [3, -0.4]]
     """
-    inputs = {'X': [x]}
-    attrs = {'alpha': alpha}
     if in_dygraph_mode():
-        outs = core.ops.leaky_relu(inputs, attrs)
-        return outs['Out'][0]
+        return core.ops.leaky_relu(x, 'alpha', alpha)
 
+    inputs = {'X': [x]}
+    attrs = {'alpha': alpha}
     helper = LayerHelper('leaky_relu', **locals())
     out = helper.create_variable_for_type_inference(dtype=x.dtype)
     helper.append_op(
@@ -9351,24 +9337,21 @@ def expand(x, expand_times, name=None):
             expanded_2 = fluid.layers.expand(data_2, expand_times=expand_times)
             # the shape of expanded_2 is [48, 56].
     """
-    inputs = {"X": [x]}
-    attrs = {}
-
     if in_dygraph_mode():
         if isinstance(expand_times, (list, tuple)):
             if utils._contain_var(expand_times):
                 raise TypeError(
                     "The type of 'expand_times' in expand must be list[int] or tuple(int) in Dygraph mode, but "
                     "received %s, which contains Variable." % type(shape))
-            attrs['expand_times'] = expand_times
         else:
             raise TypeError(
                 "The type of 'expand_times' in expand must be list[int] or tuple(int) in Dygraph mode, but "
                 "received %s." % type(shape))
 
-        outs = core.ops.expand(inputs, attrs)
-        return outs['Out'][0]
+        return core.ops.expand(x, 'expand_times', expand_times)
 
+    inputs = {"X": [x]}
+    attrs = {}
     check_variable_and_dtype(
         x, 'x', ['bool', 'float32', 'float64', 'int32', 'int64'], 'expand')
     check_type(expand_times, 'expand_times', (list, tuple, Variable), 'expand')
@@ -9908,8 +9891,6 @@ def slice(input, axes, starts, ends):
     """
     if in_dygraph_mode():
         infer_flags = list(1 for i in range(len(axes)))
-        inputs = {'Input': [input]}
-
         if isinstance(starts, (list, tuple)):
             if utils._contain_var(starts):
                 raise TypeError(
@@ -9930,14 +9911,8 @@ def slice(input, axes, starts, ends):
                 "The type of 'ends' in slice must be list[int] or tuple(int) in Dygraph mode, but "
                 "received %s." % type(shape))
 
-        attrs = {
-            'axes': axes,
-            'starts': starts,
-            'ends': ends,
-            'infer_flags': infer_flags
-        }
-        outs = core.ops.slice(inputs, attrs)
-        return outs['Out'][0]
+        return core.ops.slice(input, 'axes', axes, 'starts', starts, 'ends',
+                              ends, 'infer_flags', infer_flags)
 
     if not isinstance(starts, (list, tuple, Variable)):
         raise ValueError(
@@ -10392,6 +10367,14 @@ def scale(x, scale=1.0, bias=0.0, bias_after_scale=True, act=None, name=None):
             print(res) # [array([[ 3.,  5.,  7.], [ 9., 11., 13.]], dtype=float32)]
 
     """
+
+    if in_dygraph_mode():
+        _scale = scale.numpy().item(0) if isinstance(scale, Variable) else scale
+        out = core.ops.scale(x, 'scale',
+                             float(_scale), 'bias',
+                             float(bias), 'bias_after_scale', bias_after_scale)
+        return dygraph_utils._append_activation_in_dygraph(out)
+
     inputs = {'X': [x]}
     attrs = {
         'bias': float(bias),
@@ -10401,11 +10384,6 @@ def scale(x, scale=1.0, bias=0.0, bias_after_scale=True, act=None, name=None):
         inputs['ScaleTensor'] = [scale]
     else:
         attrs['scale'] = float(scale)
-
-    if in_dygraph_mode():
-        outs = core.ops.scale(inputs, attrs)
-        return dygraph_utils._append_activation_in_dygraph(outs['Out'][0])
-
     helper = LayerHelper('scale', **locals())
     if name is None:
         out = helper.create_variable_for_type_inference(dtype=x.dtype)
@@ -11369,9 +11347,7 @@ def mean(x, name=None):
             mean = fluid.layers.mean(input)
     """
     if in_dygraph_mode():
-        inputs = {"X": [x]}
-        outs = core.ops.mean(inputs)
-        return outs['Out'][0]
+        return core.ops.mean(x)
 
     helper = LayerHelper("mean", **locals())
     check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], 'mean')
@@ -11453,12 +11429,12 @@ def mul(x, y, x_num_col_dims=1, y_num_col_dims=1, name=None):
             
 
     """
-    inputs = {"X": [x], "Y": [y]}
-    attrs = {"x_num_col_dims": x_num_col_dims, "y_num_col_dims": y_num_col_dims}
     if in_dygraph_mode():
-        outs = core.ops.mul(inputs, attrs)
-        return outs['Out'][0]
+        return core.ops.mul(x, y, 'x_num_col_dims', x_num_col_dims,
+                            'y_num_col_dims', y_num_col_dims)
 
+    inputs = {"X": [x], "Y": [y]}
+    attrs = {"x_num_col_dims": x_num_col_dims, "y_num_col_dims": y_num_col_dims}
     helper = LayerHelper("mul", **locals())
     check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], 'mul')
     check_variable_and_dtype(y, 'y', ['float16', 'float32', 'float64'], 'mul')
diff --git a/python/paddle/fluid/layers/tensor.py b/python/paddle/fluid/layers/tensor.py
index 20874338002050576f173028a347e3c8f5cba4ae..98ab713602c578de9e6bc2cb074f71d4843f4c82 100644
--- a/python/paddle/fluid/layers/tensor.py
+++ b/python/paddle/fluid/layers/tensor.py
@@ -255,15 +255,12 @@ def concat(input, axis=0, name=None):
     """
 
     if in_dygraph_mode():
-        inputs = {'X': input}
         if isinstance(axis, Variable):
             axis = axis.numpy()
             assert axis.shape == (
                 1, ), "axis of type Variable should have shape [1]"
             axis = axis[0]
-        attrs = {'axis': axis}
-        outs = core.ops.concat(inputs, attrs)
-        return outs['Out'][0]
+        return core.ops.concat(input, 'axis', axis)
 
     if not isinstance(input, list):
         warnings.warn(
@@ -586,12 +583,13 @@ def fill_constant(shape, dtype, value, force_cpu=False, out=None):
                     shape))
         else:
             shape = list(shape.numpy().astype(int))
-        attrs['shape'] = shape
+        dtype = convert_np_dtype_to_dtype_(dtype)
         if out is None:
             out = _varbase_creator(dtype=dtype)
-        attrs['dtype'] = out.dtype
-        outputs = {'Out': [out]}
-        outs = core.ops.fill_constant({}, attrs, outputs)
+        core.ops.fill_constant(out, 'value',
+                               float(value), 'force_cpu', force_cpu, 'dtype',
+                               dtype, 'str_value', attrs['str_value'], 'shape',
+                               shape)
         out.stop_gradient = True
         return out
 
diff --git a/python/paddle/fluid/optimizer.py b/python/paddle/fluid/optimizer.py
index ff6a42a4235bcf9864ecba9baa84132110c55d81..f46a52b6663bd719870802783d9583bbae6291db 100644
--- a/python/paddle/fluid/optimizer.py
+++ b/python/paddle/fluid/optimizer.py
@@ -889,16 +889,11 @@ class SGDOptimizer(Optimizer):
 
     @no_grad
     def _append_optimize_op(self, block, param_and_grad):
+        lr = self._create_param_lr(param_and_grad)
         if framework.in_dygraph_mode():
-            inputs = {
-                "Param": [param_and_grad[0]],
-                "Grad": [param_and_grad[1]],
-                "LearningRate": [self._create_param_lr(param_and_grad)]
-            }
-            attrs = {}
-            outputs = {'ParamOut': [param_and_grad[0]]}
-            outs = core.ops.sgd(inputs, attrs, outputs)
-            return outs['ParamOut'][0]
+            core.ops.sgd(param_and_grad[0], lr, param_and_grad[1],
+                         param_and_grad[0])
+            return None
 
         assert isinstance(block, framework.Block)
         # create the optimize op
@@ -907,7 +902,7 @@ class SGDOptimizer(Optimizer):
             inputs={
                 "Param": param_and_grad[0],
                 "Grad": param_and_grad[1],
-                "LearningRate": self._create_param_lr(param_and_grad)
+                "LearningRate": lr
             },
             outputs={"ParamOut": param_and_grad[0]},
             stop_gradient=True)
@@ -1009,24 +1004,27 @@ class MomentumOptimizer(Optimizer):
 
         velocity_acc = self._get_accumulator(self._velocity_acc_str,
                                              param_and_grad[0])
-        attrs = {"mu": self._momentum, "use_nesterov": self._use_nesterov}
+        lr = self._create_param_lr(param_and_grad)
 
+        if framework.in_dygraph_mode():
+            _, _ = core.ops.momentum(param_and_grad[0], param_and_grad[1],
+                                     velocity_acc, lr, param_and_grad[0],
+                                     velocity_acc, 'mu', self._momentum,
+                                     'use_nesterov', self._use_nesterov)
+            return None
+
+        attrs = {"mu": self._momentum, "use_nesterov": self._use_nesterov}
         inputs = {
             "Param": [param_and_grad[0]],
             "Grad": [param_and_grad[1]],
             "Velocity": [velocity_acc],
-            "LearningRate": [self._create_param_lr(param_and_grad)]
+            "LearningRate": [lr]
         }
 
         outputs = {
             "ParamOut": [param_and_grad[0]],
             "VelocityOut": [velocity_acc]
         }
-
-        if framework.in_dygraph_mode():
-            core.ops.momentum(inputs, attrs, outputs)
-            return None
-
         # create the momentum optimize op
         momentum_op = block.append_op(
             type=self.type,
@@ -1849,12 +1847,27 @@ class AdamOptimizer(Optimizer):
                                               param_and_grad[0])
         beta2_pow_acc = self._get_accumulator(self._beta2_pow_acc_str,
                                               param_and_grad[0])
-
+        lr = self._create_param_lr(param_and_grad)
         # create the adam optimize op
+
+        if framework.in_dygraph_mode():
+            _beta1 = self._beta1 if not isinstance(
+                self._beta1, Variable) else self._beta1.numpy().item(0)
+            _beta2 = self._beta2 if not isinstance(
+                self._beta2, Variable) else self._beta2.numpy().item(0)
+            _, _, _, _, _ = core.ops.adam(
+                param_and_grad[0], param_and_grad[1], lr, moment1, moment2,
+                beta1_pow_acc, beta2_pow_acc, param_and_grad[0], moment1,
+                moment2, beta1_pow_acc, beta2_pow_acc, 'epsilon', self._epsilon,
+                'lazy_mode', self._lazy_mode, 'min_row_size_to_use_multithread',
+                1000, 'beta1', _beta1, 'beta2', _beta2)
+
+            return None
+
         inputs = {
             "Param": [param_and_grad[0]],
             "Grad": [param_and_grad[1]],
-            "LearningRate": [self._create_param_lr(param_and_grad)],
+            "LearningRate": [lr],
             "Moment1": [moment1],
             "Moment2": [moment2],
             "Beta1Pow": [beta1_pow_acc],
@@ -1882,10 +1895,6 @@ class AdamOptimizer(Optimizer):
         else:
             attrs['beta2'] = self._beta2
 
-        if framework.in_dygraph_mode():
-            core.ops.adam(inputs, attrs, outputs)
-            return None
-
         adam_op = block.append_op(
             type=self.type,
             inputs=inputs,
diff --git a/python/paddle/fluid/regularizer.py b/python/paddle/fluid/regularizer.py
index 439d60ad674b2587072a86bcb2252a435b997275..96b4f16c961c8776edc37c50ee4fee2841d3c8d4 100644
--- a/python/paddle/fluid/regularizer.py
+++ b/python/paddle/fluid/regularizer.py
@@ -54,7 +54,7 @@ def _create_regularization_of_grad(param, grad, regularization=None):
     inputs = {"X": [grad, regularization_term]}
     outputs = {"Out": [new_grad]}
     if in_dygraph_mode():
-        core.ops.sum(inputs, {}, outputs)
+        new_grad = core.ops.sum([grad, regularization_term])
     else:
         grad.block.append_op(type='sum', inputs=inputs, outputs=outputs)
 
@@ -183,8 +183,7 @@ class L2DecayRegularizer(WeightDecayRegularizer):
         attrs = {"scale": self._regularization_coeff}
 
         if framework.in_dygraph_mode():
-            outs = core.ops.scale(inputs, attrs)
-            return outs['Out'][0]
+            return core.ops.scale(param, "scale", self._regularization_coeff)
         else:
             decay = block.create_var(
                 dtype=param.dtype, shape=param.shape, lod_level=param.lod_level)
diff --git a/python/paddle/fluid/tests/unittests/test_imperative_star_gan_with_gradient_penalty.py b/python/paddle/fluid/tests/unittests/test_imperative_star_gan_with_gradient_penalty.py
index afb113ed941e91b0d5cdf50b938704c3f23cda44..176895aa9e7feeb35817edb469cdd11b289b7cbd 100644
--- a/python/paddle/fluid/tests/unittests/test_imperative_star_gan_with_gradient_penalty.py
+++ b/python/paddle/fluid/tests/unittests/test_imperative_star_gan_with_gradient_penalty.py
@@ -112,9 +112,9 @@ class InstanceNorm(fluid.dygraph.Layer):
 
     def forward(self, input):
         if fluid.in_dygraph_mode():
-            inputs = {'X': [input], 'Scale': [self.scale], 'Bias': [self.bias]}
-            attrs = {'epsilon': self.epsilon}
-            return fluid.core.ops.instance_norm(inputs, attrs)['Y'][0]
+            out, _, _ = fluid.core.ops.instance_norm(
+                input, self.scale, self.bias, 'epsilon', self.epsilon)
+            return out
         else:
             return fluid.layers.instance_norm(
                 input,
diff --git a/python/paddle/fluid/tests/unittests/test_op_function_generator.py b/python/paddle/fluid/tests/unittests/test_op_function_generator.py
index 72c74d23ae76b471d4f276aedad2c1beccd23be7..c59eca90d7536b566e5aebe78a1b2ccab5530a35 100644
--- a/python/paddle/fluid/tests/unittests/test_op_function_generator.py
+++ b/python/paddle/fluid/tests/unittests/test_op_function_generator.py
@@ -28,8 +28,7 @@ class TestTracedLayer(fluid.dygraph.Layer):
         super(TestTracedLayer, self).__init__(name_scope)
 
     def forward(self, input):
-        inputs = {'X': [input] if isinstance(input, fluid.Variable) else input}
-        return core.ops.relu(inputs)['Out'][0]
+        return core.ops.relu(input)
 
 
 class TestVariable(unittest.TestCase):
@@ -47,9 +46,7 @@ class TestVariable(unittest.TestCase):
             x.stop_gradient = False
 
             res1 = layers.elementwise_add(x, y)
-
-            inputs = {'X': [x], 'Y': [y]}
-            res2 = core.ops.elementwise_add(inputs)['Out'][0]
+            res2 = core.ops.elementwise_add(x, y)
 
             self.assertTrue(np.array_equal(res1.numpy(), res2.numpy()))
 
@@ -61,9 +58,7 @@ class TestVariable(unittest.TestCase):
             y = fluid.dygraph.to_variable(b)
 
             res1 = layers.elementwise_mul(x, y)
-
-            inputs = {'X': [x], 'Y': [y]}
-            res2 = core.ops.elementwise_mul(inputs)['Out'][0]
+            res2 = core.ops.elementwise_mul(x, y)
 
             self.assertTrue(np.array_equal(res1.numpy(), res2.numpy()))
 
@@ -73,9 +68,7 @@ class TestVariable(unittest.TestCase):
             x = fluid.dygraph.to_variable(a)
 
             res1 = layers.relu(x)
-
-            inputs = {'X': [x]}
-            res2 = core.ops.relu(inputs)['Out'][0]
+            res2 = core.ops.relu(x)
 
             self.assertTrue(np.array_equal(res1.numpy(), res2.numpy()))
 
@@ -88,8 +81,7 @@ class TestVariable(unittest.TestCase):
             x.stop_gradient = False
             y.stop_gradient = False
 
-            inputs = {'X': [x], 'Y': [y]}
-            loss = core.ops.elementwise_mul(inputs)['Out'][0]
+            loss = core.ops.elementwise_mul(x, y)
 
             loss.backward()
             x_grad = x.gradient()
@@ -104,7 +96,7 @@ class TestVariable(unittest.TestCase):
             a = np.random.uniform(-1, 1, self.shape).astype(self.dtype)
             x = fluid.dygraph.to_variable(a)
             res_dygraph, static_layer = TracedLayer.trace(
-                layer, inputs=[x])  # dygraph out
+                layer, inputs=x)  # dygraph out
             res_static_graph = static_layer([x])[0]
 
             self.assertTrue(