From 488b2387e28a31a42f77ff9ccf450e2bc7f1bde8 Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Tue, 31 Mar 2020 10:56:52 +0800 Subject: [PATCH] Feature/expand params in auto-generated pybind functions for dygraph operators (#23181) * expand parameters, test=develop * support resnet, test=develop * fix resnet, test=develop * support duplicable out, test=develop * support ptb * fix bugs, test=develop * support null input, test=develop * fix bugs, test=develop * fix batchNorm is_test, test=develop * refine code, test=develop * follow comments, test=develop * follow comments, test=develop * follow comments, test=develop * follow comments, test=develop --- paddle/fluid/pybind/op_function.h | 34 ++- paddle/fluid/pybind/op_function_generator.cc | 208 +++++++++++++++--- python/paddle/fluid/dygraph/math_op_patch.py | 39 +--- python/paddle/fluid/dygraph/nn.py | 151 +++++++------ python/paddle/fluid/dygraph_utils.py | 21 +- .../fluid/layers/layer_function_generator.py | 4 +- python/paddle/fluid/layers/loss.py | 37 ++-- python/paddle/fluid/layers/metric_op.py | 27 +-- python/paddle/fluid/layers/nn.py | 192 +++++++--------- python/paddle/fluid/layers/tensor.py | 14 +- python/paddle/fluid/optimizer.py | 55 +++-- python/paddle/fluid/regularizer.py | 5 +- ...perative_star_gan_with_gradient_penalty.py | 6 +- .../unittests/test_op_function_generator.py | 20 +- 14 files changed, 475 insertions(+), 338 deletions(-) diff --git a/paddle/fluid/pybind/op_function.h b/paddle/fluid/pybind/op_function.h index 07e6ff0d872..597ead9327e 100644 --- a/paddle/fluid/pybind/op_function.h +++ b/paddle/fluid/pybind/op_function.h @@ -18,13 +18,45 @@ #include #include #include +#include +#include +#include #include "paddle/fluid/framework/attribute.h" #include "paddle/fluid/framework/op_info.h" #include "paddle/fluid/framework/variable.h" #include "paddle/fluid/imperative/tracer.h" #include "paddle/fluid/imperative/type_defs.h" - #include "paddle/fluid/pybind/imperative.h" +namespace py = pybind11; +namespace paddle { +namespace pybind { +static inline void ConstructAttrMapFromPyArgs(framework::AttributeMap* attrs, + const py::args& args) { + PADDLE_ENFORCE_EQ( + args.size() % 2, 0, + platform::errors::InvalidArgument( + "The number of arguments for arributes should be even.")); + for (size_t i = 0; i < args.size(); i += 2) { + auto name = args[i].cast(); + auto value = args[i + 1].cast(); + (*attrs)[name] = value; + } +} + +static inline std::vector> +ConstructDuplicableOutput(const size_t num) { + auto tracer = imperative::GetCurrentTracer(); + std::vector> res; + res.reserve(num); + for (size_t i = 0; i < num; i++) { + auto var_base_name = tracer->GenerateUniqueName(); + res.emplace_back(new imperative::VarBase(var_base_name)); + } + return res; +} +} // namespace pybind +} // namespace paddle + // This include must be the last line #include "paddle/fluid/pybind/op_function_impl.h" diff --git a/paddle/fluid/pybind/op_function_generator.cc b/paddle/fluid/pybind/op_function_generator.cc index 2c112079ccf..da30a12c184 100644 --- a/paddle/fluid/pybind/op_function_generator.cc +++ b/paddle/fluid/pybind/op_function_generator.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include #include @@ -23,46 +24,93 @@ #include "paddle/fluid/pybind/pybind.h" #include "paddle/fluid/string/string_helper.h" +std::map> op_ins_map = { + {"layer_norm", {"X", "Scale", "Bias"}}, + {"gru_unit", {"Input", "HiddenPrev", "Weight", "Bias"}}, + {"label_smooth", {"X", "PriorDist"}}, + {"assign", {"X"}}, +}; +std::map> op_passing_out_map = { + {"sgd", {"ParamOut"}}, + {"adam", + {"ParamOut", "Moment1Out", "Moment2Out", "Beta1PowOut", "Beta2PowOut"}}, + {"momentum", {"ParamOut", "VelocityOut"}}, + {"batch_norm", {"MeanOut", "VarianceOut"}}, + {"accuracy", {"Correct", "Total"}}, + {"fill_constant", {"Out"}}}; + // clang-format off const char* OUT_INITIALIZER_TEMPLATE = R"({"%s", {std::shared_ptr(new imperative::VarBase(tracer->GenerateUniqueName()))}})"; +const char* OUT_DUPLICABLE_INITIALIZER_TEMPLATE = R"({"%s", ConstructDuplicableOutput(%s)})"; + +const char* INPUT_INITIALIZER_TEMPLATE = R"({"%s", {%s}})"; +const char* INPUT_LIST_INITIALIZER_TEMPLATE = R"({"%s", %s})"; +const char* INPUT_INITIALIZER_TEMPLATE_WITH_NULL = R"( + if (%s != nullptr) { + ins["%s"] = {%s}; + } +)"; +const char* INPUT_INITIALIZER_TEMPLATE_WITH_NULL_LIST = R"( + if (%s != nullptr) { + ins["%s"] = %s; + } +)"; + +// if inputs is list, no need {} +const char* ARG_OUT_NUM = R"(%sNum)"; +const char* ARG_OUT_NUM_TYPE = R"(size_t )"; + +const char* VAR_TYPE = R"(std::shared_ptr)"; +const char* VAR_LIST_TYPE = R"(std::vector>)"; +const char* ARG_TEMPLATE = R"(const %s& %s)"; + +const char* RETURN_TUPLE_TYPE = R"(std::tuple<%s>)"; +const char* RETURN_TYPE = R"(%s)"; +const char* RETURN_TUPLE_TEMPLATE = R"(std::make_tuple(%s))"; +const char* RETURN_LIST_TEMPLATE = R"(outs["%s"])"; +const char* RETURN_TEMPLATE = R"(outs["%s"][0])"; + +const char* FUNCTION_ARGS = R"(%s, const py::args& args)"; +const char* FUNCTION_ARGS_NO_INPUT = R"(const py::args& args)"; const char* OP_FUNCTION_TEMPLATE = R"( -inline imperative::NameVarBaseMap %s(const imperative::NameVarBaseMap& ins, const framework::AttributeMap& attrs, - imperative::NameVarBaseMap outs, const std::map& out_nums) +%s %s(%s) { - auto tracer = imperative::GetCurrentTracer(); - if (outs.size() == 0) { - if (out_nums.size() == 0) { - imperative::NameVarBaseMap outs_ = %s; - outs = std::move(outs_); - } else { - for (auto &pair : out_nums) { - for (size_t i = 0; i < pair.second; i ++) { - auto var_base_name = tracer->GenerateUniqueName(); - outs[pair.first].emplace_back(new imperative::VarBase(var_base_name)); - } - } - } - } - - tracer->TraceOp("%s", std::move(ins), std::move(outs), std::move(attrs)); - return outs; + framework::AttributeMap attrs; + ConstructAttrMapFromPyArgs(&attrs, args); + { + py::gil_scoped_release release; + auto tracer = imperative::GetCurrentTracer(); + imperative::NameVarBaseMap outs = %s; + imperative::NameVarBaseMap ins = %s; + %s + tracer->TraceOp("%s", ins, outs, attrs); + return %s; + } })"; -const char* PYBIND_ITEM_TEMPLATE = -R"( - %s.def("%s", &%s, py::arg("ins"), py::arg("attrs")=framework::AttributeMap(), py::arg("outs")=imperative::NameVarBaseMap(), - py::arg("out_nums")=std::map(), py::call_guard());)"; +const char* PYBIND_ITEM_TEMPLATE = R"( %s.def("%s", &%s);)"; // clang-format on +static inline bool FindInputInSpecialization(const std::string& op_type, + const std::string& in_name) { + return op_ins_map[op_type].count(in_name); +} + +static inline bool FindOutoutInSpecialization(const std::string& op_type, + const std::string& out_name) { + return op_passing_out_map[op_type].count(out_name); +} static std::tuple, std::vector> GenerateOpFunctions(const std::string& module_name) { auto& op_info_map = paddle::framework::OpInfoMap::Instance().map(); std::vector op_function_list, bind_function_list; + auto& all_kernels = paddle::framework::OperatorWithKernel::AllOpKernels(); + for (auto& pair : op_info_map) { auto& op_info = pair.second; auto op_proto = op_info.proto_; @@ -70,27 +118,131 @@ GenerateOpFunctions(const std::string& module_name) { continue; } auto& op_type = op_proto->type(); + // Skip ooerator which is not inherit form OperatorWithKernel, like while, + // since only OperatorWithKernel can run in dygraph mode. + if (!all_kernels.count(op_type)) { + continue; + } + std::string input_args = ""; + std::string ins_initializer = "{"; + std::string ins_initializer_with_null = ""; + std::string py_arg = ""; + for (auto& input : op_proto->inputs()) { + auto& in_name = input.name(); + // skip those dispensable inputs, like ResidualData in conv2d + if (input.dispensable() && !FindInputInSpecialization(op_type, in_name)) { + continue; + } + const auto in_type = input.duplicable() ? VAR_LIST_TYPE : VAR_TYPE; + auto input_arg = paddle::string::Sprintf(ARG_TEMPLATE, in_type, in_name); + input_args += input_arg; + input_args += ","; + + if (input.dispensable()) { + const auto in_template = input.duplicable() + ? INPUT_INITIALIZER_TEMPLATE_WITH_NULL_LIST + : INPUT_INITIALIZER_TEMPLATE_WITH_NULL; + ins_initializer_with_null += + paddle::string::Sprintf(in_template, in_name, in_name, in_name); + } else { + const auto in_template = input.duplicable() + ? INPUT_LIST_INITIALIZER_TEMPLATE + : INPUT_INITIALIZER_TEMPLATE; + ins_initializer += + paddle::string::Sprintf(in_template, in_name, in_name); + ins_initializer += ","; + } + } + if (ins_initializer.back() == ',') { + ins_initializer.pop_back(); + } + ins_initializer += "}"; + + if (input_args.back() == ',') { + input_args.pop_back(); + } // Generate outs initializer std::string outs_initializer = "{"; + std::string return_type = ""; + std::string return_str = ""; + int outs_num = 0; for (auto& output : op_proto->outputs()) { + if (output.dispensable()) { + continue; + } + const auto out_type = output.duplicable() ? VAR_LIST_TYPE : VAR_TYPE; + const auto return_template = + output.duplicable() ? RETURN_LIST_TEMPLATE : RETURN_TEMPLATE; auto& out_name = output.name(); - auto out_initializer_str = - paddle::string::Sprintf(OUT_INITIALIZER_TEMPLATE, out_name); + std::string out_initializer_str; + if (FindOutoutInSpecialization(op_type, out_name)) { + if (input_args != "") { + input_args += ","; + } + input_args += out_type; + input_args += out_name; + const auto out_template = output.duplicable() + ? INPUT_LIST_INITIALIZER_TEMPLATE + : INPUT_INITIALIZER_TEMPLATE; + out_initializer_str += + paddle::string::Sprintf(out_template, out_name, out_name); + } else { + // There are few Operators that have duplicable output, like `Out` in + // split op. We need to specify the number of variables for the + // duplicable output, as the argument OutNum; + if (output.duplicable()) { + if (input_args != "") { + input_args += ","; + } + auto out_num_str = paddle::string::Sprintf(ARG_OUT_NUM, out_name); + input_args += ARG_OUT_NUM_TYPE; + input_args += out_num_str; + out_initializer_str = paddle::string::Sprintf( + OUT_DUPLICABLE_INITIALIZER_TEMPLATE, out_name, out_num_str); + } else { + out_initializer_str = + paddle::string::Sprintf(OUT_INITIALIZER_TEMPLATE, out_name); + } + } + + return_type += out_type; + return_type += ","; + return_str += paddle::string::Sprintf(return_template, out_name); + return_str += ","; + outs_num += 1; + outs_initializer += out_initializer_str; outs_initializer += ","; } if (outs_initializer.back() == ',') { outs_initializer.pop_back(); + return_type.pop_back(); + return_str.pop_back(); } outs_initializer += "}"; + if (outs_num == 0) { + return_type = "void"; + } + if (outs_num > 1) { + return_str = paddle::string::Sprintf(RETURN_TUPLE_TEMPLATE, return_str); + return_type = paddle::string::Sprintf(RETURN_TUPLE_TYPE, return_type); + } + std::string function_args = ""; + if (input_args == "") { + function_args = + paddle::string::Sprintf(FUNCTION_ARGS_NO_INPUT, input_args); + } else { + function_args = paddle::string::Sprintf(FUNCTION_ARGS, input_args); + } std::string func_name = "imperative_" + op_type; - // generate op funtcion body auto op_function_str = paddle::string::Sprintf( - OP_FUNCTION_TEMPLATE, func_name, outs_initializer, op_type); + OP_FUNCTION_TEMPLATE, return_type, func_name, function_args, + outs_initializer, ins_initializer, ins_initializer_with_null, op_type, + return_str); // generate pybind item auto bind_function_str = paddle::string::Sprintf( @@ -99,7 +251,6 @@ GenerateOpFunctions(const std::string& module_name) { op_function_list.emplace_back(std::move(op_function_str)); bind_function_list.emplace_back(std::move(bind_function_str)); } - return std::make_tuple(op_function_list, bind_function_list); } @@ -119,7 +270,6 @@ int main(int argc, char* argv[]) { out << "#include " + header + "\n"; } - // all op functions auto op_funcs = GenerateOpFunctions("m"); out << "namespace py = pybind11;" diff --git a/python/paddle/fluid/dygraph/math_op_patch.py b/python/paddle/fluid/dygraph/math_op_patch.py index 0424781cf3e..dbd8d803f66 100644 --- a/python/paddle/fluid/dygraph/math_op_patch.py +++ b/python/paddle/fluid/dygraph/math_op_patch.py @@ -15,7 +15,7 @@ from __future__ import print_function from .. import core -from ..framework import Variable, convert_np_dtype_to_dtype_ +from ..framework import Variable, convert_np_dtype_to_dtype_, _varbase_creator from ..layers.layer_function_generator import OpProtoHolder from . import to_variable, no_grad @@ -42,17 +42,11 @@ def monkey_patch_math_varbase(): @no_grad def create_tensor(value, dtype, shape): - value = float(value) - inputs = {} - attrs = { - 'dtype': dtype, - 'shape': shape, - 'value': value, - 'force_cpu': False - } - outs = core.ops.fill_constant(inputs, attrs) - outs['Out'][0].stop_gradient = True - return outs['Out'][0] + out = _varbase_creator(dtype=dtype) + out = core.ops.fill_constant(out, 'dtype', dtype, 'shape', shape, + 'value', value, 'force_cpu', False) + out.stop_gradient = True + return out def create_scalar(value, dtype): return create_tensor(value, dtype, shape=[1]) @@ -102,19 +96,11 @@ def monkey_patch_math_varbase(): print("new var's dtype is: {}, numpy dtype is {}".format(new_variable.dtype, new_variable.numpy().dtype)) """ - inputs = {'X': [self]} - attrs = { - "in_dtype": self.dtype, - "out_dtype": convert_np_dtype_to_dtype_(dtype) - } - outs = core.ops.cast(inputs, attrs) - return outs['Out'][0] + return core.ops.cast(self, 'in_dtype', self.dtype, 'out_dtype', + convert_np_dtype_to_dtype_(dtype)) def _scalar_elementwise_op_(var, scale, bias): - inputs = {'X': [var]} - attrs = {"scale": scale, "bias": bias} - outs = core.ops.scale(inputs, attrs) - return outs['Out'][0] + return core.ops.scale(var, 'scale', scale, 'bias', bias) def _neg_(var): return _scalar_elementwise_op_(var, -1.0, 0.0) @@ -208,11 +194,8 @@ def monkey_patch_math_varbase(): other_var = tmp axis = -1 - op = getattr(core.ops, op_type) - inputs = {'X': [self], 'Y': [other_var]} - attrs = {'axis': axis} - outs = op(inputs, attrs) - return outs['Out'][0] + math_op = getattr(core.ops, op_type) + return math_op(self, other_var, 'aixs', axis) comment = OpProtoHolder.instance().get_op_proto(op_type).comment diff --git a/python/paddle/fluid/dygraph/nn.py b/python/paddle/fluid/dygraph/nn.py index c9ede3bdefe..742e29b0ba0 100644 --- a/python/paddle/fluid/dygraph/nn.py +++ b/python/paddle/fluid/dygraph/nn.py @@ -218,6 +218,17 @@ class Conv2D(layers.Layer): is_bias=True) def forward(self, input): + if in_dygraph_mode() and self._l_type == 'conv2d': + attrs = ('strides', self._stride, 'paddings', self._padding, + 'dilations', self._dilation, 'groups', self._groups + if self._groups else 1, 'use_cudnn', self._use_cudnn) + out = core.ops.conv2d(input, self.weight, *attrs) + pre_bias = out + + pre_act = dygraph_utils._append_bias_in_dygraph(pre_bias, self.bias, + 1) + return dygraph_utils._append_activation_in_dygraph(pre_act, + self._act) inputs = { 'Input': [input], 'Filter': [self.weight], @@ -230,17 +241,6 @@ class Conv2D(layers.Layer): 'use_cudnn': self._use_cudnn, 'use_mkldnn': False, } - - if in_dygraph_mode() and self._l_type == 'conv2d': - outs = core.ops.conv2d(inputs, attrs) - pre_bias = outs['Output'][0] - - pre_act = dygraph_utils._append_bias_in_dygraph(pre_bias, self.bias, - 1) - - return dygraph_utils._append_activation_in_dygraph(pre_act, - self._act) - pre_bias = self._helper.create_variable_for_type_inference( dtype=self._dtype) @@ -825,6 +825,14 @@ class Pool2D(layers.Layer): self._l_type = 'pool2d' def forward(self, input): + if in_dygraph_mode(): + attrs = ('pooling_type', self._pool_type, 'ksize', self._pool_size, + 'global_pooling', self._global_pooling, 'strides', + self._pool_stride, 'paddings', self._pool_padding, + 'use_cudnn', self._use_cudnn, 'ceil_mode', self._ceil_mode, + 'use_mkldnn', False, 'exclusive', self._exclusive) + return core.ops.pool2d(input, *attrs) + attrs = { "pooling_type": self._pool_type, "ksize": self._pool_size, @@ -838,10 +846,6 @@ class Pool2D(layers.Layer): } inputs = {"X": [input]} - if in_dygraph_mode(): - outs = core.ops.pool2d(inputs, attrs) - return outs['Out'][0] - pool_out = self._helper.create_variable_for_type_inference(self._dtype) self._helper.append_op( @@ -922,21 +926,20 @@ class Linear(layers.Layer): shape=[output_dim], attr=bias_attr, dtype=dtype, is_bias=True) def forward(self, input): - attrs = { - "x_num_col_dims": len(input.shape) - 1, - "y_num_col_dims": 1, - } - inputs = {"X": [input], "Y": [self.weight]} - if in_dygraph_mode(): - outs = core.ops.mul(inputs, attrs) - pre_bias = outs['Out'][0] + pre_bias = core.ops.mul(input, self.weight, 'x_num_col_dims', + len(input.shape) - 1, 'y_num_col_dims', 1) pre_act = dygraph_utils._append_bias_in_dygraph( pre_bias, self.bias, axis=len(input.shape) - 1) return dygraph_utils._append_activation_in_dygraph(pre_act, self._act) + attrs = { + "x_num_col_dims": len(input.shape) - 1, + "y_num_col_dims": 1, + } + inputs = {"X": [input], "Y": [self.weight]} tmp = self._helper.create_variable_for_type_inference(self._dtype) self._helper.append_op( @@ -1128,8 +1131,22 @@ class BatchNorm(layers.Layer): # mean and mean_out share the same memory mean_out = self._mean # variance and variance out share the same memory - variance_out = self._variance + + if in_dygraph_mode(): + _is_test = (not _dygraph_tracer()._train_mode) and ( + not self._trainable_statistics) + attrs = ("momentum", self._momentum, "epsilon", self._epsilon, + "is_test", _is_test, "data_layout", self._data_layout, + "use_mkldnn", False, "fuse_with_relu", + self._fuse_with_relu, "use_global_stats", + self._use_global_stats) + batch_norm_out, _, _, _, _ = core.ops.batch_norm( + input, self.weight, self.bias, self._mean, self._variance, + mean_out, variance_out, *attrs) + return dygraph_utils._append_activation_in_dygraph( + batch_norm_out, act=self._act) + attrs = { "momentum": self._momentum, "epsilon": self._epsilon, @@ -1149,20 +1166,12 @@ class BatchNorm(layers.Layer): "Variance": [self._variance] } - if in_dygraph_mode(): - attrs['is_test'] = not _dygraph_tracer()._train_mode - saved_mean = _varbase_creator(dtype=self._dtype) - saved_variance = _varbase_creator(dtype=self._dtype) - batch_norm_out = _varbase_creator(dtype=self._dtype) - batch_norm_out.stop_gradient = False - # inplace is not supported currently - else: - saved_mean = self._helper.create_variable_for_type_inference( - dtype=self._dtype, stop_gradient=True) - saved_variance = self._helper.create_variable_for_type_inference( - dtype=self._dtype, stop_gradient=True) - batch_norm_out = input if self._in_place else self._helper.create_variable_for_type_inference( - self._dtype) + saved_mean = self._helper.create_variable_for_type_inference( + dtype=self._dtype, stop_gradient=True) + saved_variance = self._helper.create_variable_for_type_inference( + dtype=self._dtype, stop_gradient=True) + batch_norm_out = input if self._in_place else self._helper.create_variable_for_type_inference( + self._dtype) outputs = { "Y": [batch_norm_out], @@ -1172,11 +1181,6 @@ class BatchNorm(layers.Layer): "SavedVariance": [saved_variance] } - if in_dygraph_mode(): - outs = core.ops.batch_norm(inputs, attrs, outputs) - return dygraph_utils._append_activation_in_dygraph( - batch_norm_out, act=self._act) - self._helper.append_op( type="batch_norm", inputs=inputs, outputs=outputs, attrs=attrs) @@ -1315,6 +1319,12 @@ class Embedding(layers.Layer): is_bias=False) def forward(self, input): + if in_dygraph_mode(): + return core.ops.lookup_table_v2( + self.weight, input, 'is_sparse', self._is_sparse, + 'is_distributed', self._is_distributed, 'remote_prefetch', + self._remote_prefetch, 'padding_idx', self._padding_idx) + attrs = { 'is_sparse': self._is_sparse, 'is_distributed': self._is_distributed, @@ -1322,11 +1332,6 @@ class Embedding(layers.Layer): 'padding_idx': self._padding_idx } - if in_dygraph_mode(): - inputs = {'Ids': [input], 'W': [self.weight]} - outs = core.ops.lookup_table_v2(inputs, attrs) - return outs['Out'][0] - out = self._helper.create_variable_for_type_inference(self._dtype) self._helper.append_op( type='lookup_table_v2', @@ -1436,6 +1441,7 @@ class LayerNorm(layers.Layer): else: if self._param_attr: logging.warn("param_attr are only available with scale is True") + self.weight = None if self._shift: assert self._bias_attr is not False @@ -1447,6 +1453,7 @@ class LayerNorm(layers.Layer): else: if self._bias_attr: logging.warn("bias_attr are only available with shift is True") + self.bias = None def forward(self, input): input_shape = list(input.shape) @@ -1460,24 +1467,25 @@ class LayerNorm(layers.Layer): 'Given normalized_shape is ' + str_normalized_shape + ', expected input with shape [*, ' + str_normalized_shape[ 1:] + ', but got input shape ' + str(input_shape)) + + if in_dygraph_mode(): + pre_act, _, _ = core.ops.layer_norm( + input, self.weight, self.bias, 'epsilon', self._epsilon, + 'begin_norm_axis', self._begin_norm_axis) + return dygraph_utils._append_activation_in_dygraph( + pre_act, act=self._act) + inputs = dict() inputs['X'] = [input] if self._scale: inputs['Scale'] = [self.weight] if self._shift: inputs['Bias'] = [self.bias] - attrs = { "epsilon": self._epsilon, "begin_norm_axis": self._begin_norm_axis } - if in_dygraph_mode(): - outs = core.ops.layer_norm(inputs, attrs) - pre_act = outs['Y'][0] - return dygraph_utils._append_activation_in_dygraph( - pre_act, act=self._act) - # create output mean_out = self._helper.create_variable_for_type_inference( dtype=self._dtype, stop_gradient=True) @@ -1642,6 +1650,12 @@ class GRUUnit(layers.Layer): attr=bias_attr, shape=bias_size, dtype=dtype, is_bias=True) def forward(self, input, hidden): + if in_dygraph_mode(): + gate, reset_hidden_pre, updated_hidden = core.ops.gru_unit( + input, hidden, self.weight, self.bias, 'activation', + self.activation, 'gate_activation', self.gate_activation) + return updated_hidden, reset_hidden_pre, gate + inputs = { 'Input': [input], 'HiddenPrev': [hidden], @@ -1653,12 +1667,6 @@ class GRUUnit(layers.Layer): 'activation': self.activation, 'gate_activation': self.gate_activation, } - - if in_dygraph_mode(): - outs = core.ops.gru_unit(inputs, attrs) - return outs['Hidden'][0], outs['ResetHiddenPrev'][0], outs['Gate'][ - 0] - gate = self._helper.create_variable_for_type_inference(self._dtype) reset_hidden_pre = self._helper.create_variable_for_type_inference( self._dtype) @@ -2310,6 +2318,18 @@ class Conv2DTranspose(layers.Layer): is_bias=True) def forward(self, input): + if in_dygraph_mode(): + op = getattr(core.ops, self._op_type) + out = op(input, self.weight, 'output_size', self._output_size, + 'strides', self._stride, 'paddings', self._padding, + 'dilations', self._dilation, 'groups', self._groups, + 'use_cudnn', self._use_cudnn) + pre_bias = out + pre_act = dygraph_utils._append_bias_in_dygraph(pre_bias, self.bias, + 1) + return dygraph_utils._append_activation_in_dygraph( + pre_act, act=self._act) + inputs = {'Input': [input], 'Filter': [self.weight]} attrs = { 'output_size': self._output_size, @@ -2320,15 +2340,6 @@ class Conv2DTranspose(layers.Layer): 'use_cudnn': self._use_cudnn } - if in_dygraph_mode(): - op = getattr(core.ops, self._op_type) - outs = op(inputs, attrs) - pre_bias = outs['Output'][0] - pre_act = dygraph_utils._append_bias_in_dygraph(pre_bias, self.bias, - 1) - return dygraph_utils._append_activation_in_dygraph( - pre_act, act=self._act) - pre_bias = self._helper.create_variable_for_type_inference( dtype=input.dtype) self._helper.append_op( diff --git a/python/paddle/fluid/dygraph_utils.py b/python/paddle/fluid/dygraph_utils.py index f8faa3d55b7..6ab3c27fa4c 100644 --- a/python/paddle/fluid/dygraph_utils.py +++ b/python/paddle/fluid/dygraph_utils.py @@ -33,15 +33,15 @@ def _append_activation_in_dygraph(input, """ if not act: return input - attrs = {} - if (use_cudnn is not None) and use_cudnn: - attrs['use_cudnn'] = use_cudnn - if (use_mkldnn is not None) and use_mkldnn: - attrs['use_mkldnn'] = use_mkldnn - inputs = {"X": [input]} + + attrs = () + if use_cudnn: + attrs = ('use_cudnn', use_cudnn) + if use_mkldnn: + attrs += ('use_mkldnn', use_mkldnn) + act_op = getattr(core.ops, act) - res = act_op(inputs, attrs) - return res['Out'][0] + return act_op(input, *attrs) @dygraph_only @@ -58,7 +58,4 @@ def _append_bias_in_dygraph(input, bias=None, axis=1): if not bias: return input - attrs = {'axis': axis} - inputs = {'X': [input], 'Y': [bias]} - outs = core.ops.elementwise_add(inputs, attrs) - return outs['Out'][0] + return core.ops.elementwise_add(input, bias, 'axis', axis) diff --git a/python/paddle/fluid/layers/layer_function_generator.py b/python/paddle/fluid/layers/layer_function_generator.py index d113868fbc9..511f1274db0 100755 --- a/python/paddle/fluid/layers/layer_function_generator.py +++ b/python/paddle/fluid/layers/layer_function_generator.py @@ -253,10 +253,8 @@ def generate_activation_fn(op_type): def func(x, name=None): if in_dygraph_mode(): - inputs = {'X': [x]} op = getattr(core.ops, op_type) - outs = op(inputs) - return outs['Out'][0] + return op(x) check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], op_type) diff --git a/python/paddle/fluid/layers/loss.py b/python/paddle/fluid/layers/loss.py index 23c062a419b..5eda55d76a6 100644 --- a/python/paddle/fluid/layers/loss.py +++ b/python/paddle/fluid/layers/loss.py @@ -238,13 +238,13 @@ def cross_entropy(input, label, soft_label=False, ignore_index=kIgnoreIndex): if not soft_label: return cross_entropy2(input, label, ignore_index) + if in_dygraph_mode(): + return core.ops.cross_entropy(input, label, "soft_label", soft_label, + "ignore_index", ignore_index) + inputs = {'X': [input], 'Label': [label]} attrs = {"soft_label": soft_label, "ignore_index": ignore_index} - if in_dygraph_mode(): - outs = core.ops.cross_entropy(inputs, attrs) - return outs['Y'][0] - check_variable_and_dtype(input, 'input', ['float16', 'float32', 'float64'], 'cross_entropy') helper = LayerHelper('cross_entropy', **locals()) @@ -255,13 +255,13 @@ def cross_entropy(input, label, soft_label=False, ignore_index=kIgnoreIndex): def cross_entropy2(input, label, ignore_index=kIgnoreIndex): - inputs = {'X': [input], 'Label': [label]} - attrs = {'ignore_index': ignore_index} - if in_dygraph_mode(): - outs = core.ops.cross_entropy2(inputs, attrs) - return outs['Y'][0] + loss, _, _ = core.ops.cross_entropy2(input, label, 'ignore_index', + ignore_index) + return loss + inputs = {'X': [input], 'Label': [label]} + attrs = {'ignore_index': ignore_index} check_variable_and_dtype(input, 'input', ['float16', 'float32', 'float64'], 'cross_entropy2') helper = LayerHelper('cross_entropy2', **locals()) @@ -1233,21 +1233,22 @@ def softmax_with_cross_entropy(logits, out = fluid.layers.softmax_with_cross_entropy( logits=fc, label=label) """ + if in_dygraph_mode(): + softmax, loss = core.ops.softmax_with_cross_entropy( + logits, label, 'soft_label', soft_label, 'ignore_index', + ignore_index, 'numeric_stable_mode', numeric_stable_mode, 'axis', + axis) + if not return_softmax: + return loss + else: + return loss, softmax + attrs = { 'soft_label': soft_label, 'ignore_index': ignore_index, 'numeric_stable_mode': numeric_stable_mode, 'axis': axis } - - if in_dygraph_mode(): - inputs = {'Logits': [logits], 'Label': [label]} - outs = core.ops.softmax_with_cross_entropy(inputs, attrs) - if not return_softmax: - return outs['Loss'][0] - else: - return outs['Loss'][0], outs['Softmax'][0] - helper = LayerHelper('softmax_with_cross_entropy', **locals()) softmax = helper.create_variable_for_type_inference(dtype=logits.dtype) loss = helper.create_variable_for_type_inference(dtype=logits.dtype) diff --git a/python/paddle/fluid/layers/metric_op.py b/python/paddle/fluid/layers/metric_op.py index de92f3c3cfe..3b6207c2c60 100644 --- a/python/paddle/fluid/layers/metric_op.py +++ b/python/paddle/fluid/layers/metric_op.py @@ -74,24 +74,15 @@ def accuracy(input, label, k=1, correct=None, total=None): #[array([0.6666667], dtype=float32)] """ if in_dygraph_mode(): - topk_out, topk_indices = nn.topk(input, k=k) - inputs = { - "Out": [topk_out], - "Indices": [topk_indices], - "Label": [label] - } - acc_out = _varbase_creator(dtype="float32") if correct is None: - correct = _varbase_creator(dtype="int64") + correct = _varbase_creator(dtype="int32") if total is None: - total = _varbase_creator(dtype="int64") - outputs = { - "Accuracy": [acc_out], - "Correct": [correct], - "Total": [total] - } - outs = core.ops.accuracy(inputs, {}, outputs) - return outs['Accuracy'][0] + total = _varbase_creator(dtype="int32") + + topk_out, topk_indices = nn.topk(input, k=k) + _acc, _, _ = core.ops.accuracy(topk_out, topk_indices, label, correct, + total) + return _acc helper = LayerHelper("accuracy", **locals()) check_variable_and_dtype(input, 'input', ['float16', 'float32', 'float64'], @@ -99,9 +90,9 @@ def accuracy(input, label, k=1, correct=None, total=None): topk_out, topk_indices = nn.topk(input, k=k) acc_out = helper.create_variable_for_type_inference(dtype="float32") if correct is None: - correct = helper.create_variable_for_type_inference(dtype="int64") + correct = helper.create_variable_for_type_inference(dtype="int32") if total is None: - total = helper.create_variable_for_type_inference(dtype="int64") + total = helper.create_variable_for_type_inference(dtype="int32") helper.append_op( type="accuracy", inputs={ diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index f7f1c867b66..063172a149f 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -192,11 +192,8 @@ def _elementwise_op_in_dygraph(x, act=None, use_mkldnn=False, op_name=None): - attrs = {'axis': axis, 'use_mkldnn': use_mkldnn} - inputs = {'X': [x], 'Y': [y]} op = getattr(core.ops, op_name) - outs = op(inputs, attrs) - out = outs['Out'][0] + out = op(x, y, 'axis', axis, 'use_mkldnn', use_mkldnn) return dygraph_utils._append_activation_in_dygraph( out, act, use_mkldnn=use_mkldnn) @@ -835,11 +832,16 @@ def dropout(x, return attrs if in_dygraph_mode(): - attrs = get_attrs(default_main_program(), dropout_prob, is_test, seed) - attrs['is_test'] = not _dygraph_tracer()._train_mode - inputs = {'X': [x]} - outs = core.ops.dropout(inputs, attrs) - return outs['Out'][0] + if (seed is None or + seed == 0) and default_main_program().random_seed != 0: + seed = default_main_program().random_seed + seed = seed if seed is not None else 0 + _is_test = not _dygraph_tracer()._train_mode + out, mask = core.ops.dropout(x, 'dropout_prob', dropout_prob, 'is_test', + _is_test, 'fix_seed', seed is not None, + 'seed', seed, 'dropout_implementation', + dropout_implementation) + return out helper = LayerHelper('dropout', **locals()) check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], @@ -1118,12 +1120,12 @@ def softmax(input, use_cudnn=False, name=None, axis=-1): fetch_list=[result[0]]) print(output) """ - inputs = {"X": [input]} - attrs = {"axis": axis, "use_cudnn": use_cudnn} if in_dygraph_mode(): - outs = core.ops.softmax(inputs, attrs) - return outs['Out'][0] + return core.ops.softmax(input, 'axis', axis, 'use_cudnn', use_cudnn) + + inputs = {"X": [input]} + attrs = {"axis": axis, "use_cudnn": use_cudnn} helper = LayerHelper('softmax', **locals()) check_variable_and_dtype(input, 'input', ['float16', 'float32', 'float64'], @@ -3887,17 +3889,17 @@ def reduce_sum(input, dim=None, keep_dim=False, name=None): """ if dim is not None and not isinstance(dim, list): dim = [dim] + + if in_dygraph_mode(): + reduce_all = True if dim == None or dim == [] else False + dim = dim if dim != None and dim != [] else [0] + return core.ops.reduce_sum(input, 'dim', dim, 'keep_dim', keep_dim, + 'reduce_all', reduce_all) attrs = { 'dim': dim if dim != None and dim != [] else [0], 'keep_dim': keep_dim, 'reduce_all': True if dim == None or dim == [] else False } - - if in_dygraph_mode(): - inputs = {'X': [input]} - outs = core.ops.reduce_sum(inputs, attrs) - return outs['Out'][0] - check_variable_and_dtype( input, 'input', ['float32', 'float64', 'int32', 'int64'], 'reduce_sum') helper = LayerHelper('reduce_sum', **locals()) @@ -3962,17 +3964,17 @@ def reduce_mean(input, dim=None, keep_dim=False, name=None): if dim is not None and not isinstance(dim, list): dim = [dim] + + if in_dygraph_mode(): + reduce_all = True if dim == None or dim == [] else False + dim = dim if dim != None and dim != [] else [0] + return core.ops.reduce_mean(input, 'dim', dim, 'keep_dim', keep_dim, + 'reduce_all', reduce_all) attrs = { 'dim': dim if dim != None and dim != [] else [0], 'keep_dim': keep_dim, 'reduce_all': True if dim == None or dim == [] else False } - - if in_dygraph_mode(): - inputs = {'X': [input]} - outs = core.ops.reduce_mean(inputs, attrs) - return outs['Out'][0] - check_variable_and_dtype( input, 'input', ['float32', 'float64', 'int32', 'int64'], 'reduce_mean') helper = LayerHelper('reduce_mean', **locals()) @@ -4333,19 +4335,20 @@ def split(input, num_or_sections, dim=-1, name=None): # x2.shape [3, 4, 5] """ if in_dygraph_mode(): - inputs = {'X': [input]} - attrs = {} + num = None + attrs = () + if isinstance(dim, Variable): dim = dim.numpy() assert dim.shape == (1, ), "dim of type Variable should have shape [1]" dim = dim[0] dim = (len(input.shape) + dim) if dim < 0 else dim - attrs['axis'] = dim + attrs += ('axis', dim) if isinstance(num_or_sections, int): num = num_or_sections - attrs['num'] = num_or_sections + attrs += ('num', num_or_sections) elif isinstance(num_or_sections, (list, tuple)): num = len(num_or_sections) if utils._contain_var(num_or_sections): @@ -4354,14 +4357,12 @@ def split(input, num_or_sections, dim=-1, name=None): "received %s, which contains Variable." % (type(num_or_sections))) else: - attrs['sections'] = list(num_or_sections) + attrs += ('sections', list(num_or_sections)) else: raise TypeError( "The type of 'num_or_sections' in split must be int or list in Dygraph mode, but " "received %s." % (type(num_or_sections))) - - res = core.ops.split(inputs, attrs, {}, {'Out': num}) - return res['Out'] + return core.ops.split(input, num, *attrs) if not isinstance(num_or_sections, (int, list, tuple)): raise TypeError( @@ -4596,9 +4597,8 @@ def matmul(x, y, transpose_x=False, transpose_y=False, alpha=1.0, name=None): } if in_dygraph_mode(): - inputs = {'X': [x], 'Y': [y]} - outs = core.ops.matmul(inputs, attrs) - return outs['Out'][0] + return core.ops.matmul(x, y, 'transpose_X', transpose_x, 'transpose_Y', + transpose_y, 'alpha', float(alpha)) def __check_input(x, y): var_names = {'x': x, 'y': y} @@ -4716,20 +4716,15 @@ def topk(input, k, name=None): vk_values, vk_indices = layers.topk(input2, k=vk) #vk_values.shape=[None, 13, k], vk_indices.shape=[None, 13, k] """ - inputs = {"X": [input]} - attrs = {} - if in_dygraph_mode(): - if isinstance(k, Variable): - k = k.numpy() - assert k.shape == (1, ), "k of type Variable should have shape [1]" - k = k[0] - attrs = {'k': k} - outs = core.ops.top_k(inputs, attrs) - outs['Out'][0].stop_gradient = True - outs['Indices'][0].stop_gradient = True - return outs['Out'][0], outs['Indices'][0] + _k = k.numpy().item(0) if isinstance(k, Variable) else k + out, indices = core.ops.top_k(input, 'k', _k) + out.stop_gradient = True + indices.stop_gradient = True + return out, indices + inputs = {"X": [input]} + attrs = {} if isinstance(k, Variable): inputs['K'] = [k] else: @@ -4965,10 +4960,8 @@ def transpose(x, perm, name=None): """ if in_dygraph_mode(): - attrs = {'axis': perm} - inputs = {'X': [x]} - outs = core.ops.transpose2(inputs, attrs) - return outs['Out'][0] + out, _ = core.ops.transpose2(x, 'axis', perm) + return out check_variable_and_dtype( x, 'x', ['float16', 'float32', 'float64', 'int32', 'int64'], @@ -5413,11 +5406,10 @@ def one_hot(input, depth, allow_out_of_range=False): assert depth.shape == ( 1, ), "depth of type Variable should have shape [1]" depth = depth[0] - inputs = {'X': [input]} - attrs = {'depth': depth, 'allow_out_of_range': allow_out_of_range} - outs = core.ops.one_hot(inputs, attrs) - outs['Out'][0].stop_gradient = True - return outs['Out'][0] + out = core.ops.one_hot(input, 'depth', depth, 'allow_out_of_range', + allow_out_of_range) + out.stop_gradient = True + return out helper = LayerHelper("one_hot", **locals()) one_hot_out = helper.create_variable_for_type_inference(dtype='float32') @@ -5597,9 +5589,7 @@ def reshape(x, shape, actual_shape=None, act=None, inplace=False, name=None): "The type of 'shape' in reshape must be list[int] or tuple(int) in Dygraph mode, but " "received %s." % type(shape)) - inputs = {'X': [x]} - outs = core.ops.reshape2(inputs, attrs) - out = outs['Out'][0] + out, _ = core.ops.reshape2(x, 'shape', shape) return dygraph_utils._append_activation_in_dygraph(out, act) check_variable_and_dtype( @@ -6274,12 +6264,8 @@ def label_smooth(label, raise ValueError("The value of epsilon must be between 0 and 1.") if in_dygraph_mode(): - inputs = {"X": [label]} - if prior_dist: - inputs["PriorDist"] = [prior_dist] - attrs = {"epsilon": float(epsilon)} - outs = core.ops.label_smooth(inputs, attrs) - return outs['Out'][0] + return core.ops.label_smooth(label, prior_dist, 'epsilon', + float(epsilon)) helper = LayerHelper("label_smooth", **locals()) label.stop_gradient = True @@ -7854,11 +7840,10 @@ def log(x, name=None): res_val, = exe.run(fluid.default_main_program(), feed={'x':x_i}, fetch_list=[res]) print(res_val) # [[0.], [0.6931472]] """ - inputs = {'X': [x]} if in_dygraph_mode(): - outs = core.ops.log(inputs) - return outs['Out'][0] + return core.ops.log(x) + inputs = {'X': [x]} helper = LayerHelper('log', **locals()) dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference(dtype) @@ -7894,11 +7879,10 @@ def relu(x, name=None): # [[0. 0. ] # [1. 2.6]] """ - inputs = {'X': [x]} if in_dygraph_mode(): - outs = core.ops.relu(inputs) - return outs['Out'][0] + return core.ops.relu(x) + inputs = {'X': [x]} helper = LayerHelper('relu', **locals()) dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference(dtype) @@ -8480,6 +8464,13 @@ def pad2d(input, result = fluid.layers.pad2d(input=data, paddings=[1, 2, 3, 4], mode='reflect') """ + + if in_dygraph_mode(): + _paddings = paddings.numpy().tolist() if isinstance( + paddings, Variable) else paddings + return core.ops.pad2d(input, 'mode', mode, 'pad_value', pad_value, + 'data_format', data_format, 'paddings', _paddings) + attrs = {'mode': mode, 'pad_value': pad_value, 'data_format': data_format} inputs = {'X': [input]} if isinstance(paddings, Variable): @@ -8488,10 +8479,6 @@ def pad2d(input, else: attrs['paddings'] = paddings - if in_dygraph_mode(): - outs = core.ops.pad2d(inputs, attrs) - return outs['Out'][0] - helper = LayerHelper('pad2d', **locals()) assert mode in ['reflect', 'edge', 'constant' @@ -8927,12 +8914,11 @@ def leaky_relu(x, alpha=0.02, name=None): res_val, = exe.run(fluid.default_main_program(), feed={'x':x_i}, fetch_list=[res]) print(res_val) # [[-0.1, 2], [3, -0.4]] """ - inputs = {'X': [x]} - attrs = {'alpha': alpha} if in_dygraph_mode(): - outs = core.ops.leaky_relu(inputs, attrs) - return outs['Out'][0] + return core.ops.leaky_relu(x, 'alpha', alpha) + inputs = {'X': [x]} + attrs = {'alpha': alpha} helper = LayerHelper('leaky_relu', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) helper.append_op( @@ -9351,24 +9337,21 @@ def expand(x, expand_times, name=None): expanded_2 = fluid.layers.expand(data_2, expand_times=expand_times) # the shape of expanded_2 is [48, 56]. """ - inputs = {"X": [x]} - attrs = {} - if in_dygraph_mode(): if isinstance(expand_times, (list, tuple)): if utils._contain_var(expand_times): raise TypeError( "The type of 'expand_times' in expand must be list[int] or tuple(int) in Dygraph mode, but " "received %s, which contains Variable." % type(shape)) - attrs['expand_times'] = expand_times else: raise TypeError( "The type of 'expand_times' in expand must be list[int] or tuple(int) in Dygraph mode, but " "received %s." % type(shape)) - outs = core.ops.expand(inputs, attrs) - return outs['Out'][0] + return core.ops.expand(x, 'expand_times', expand_times) + inputs = {"X": [x]} + attrs = {} check_variable_and_dtype( x, 'x', ['bool', 'float32', 'float64', 'int32', 'int64'], 'expand') check_type(expand_times, 'expand_times', (list, tuple, Variable), 'expand') @@ -9908,8 +9891,6 @@ def slice(input, axes, starts, ends): """ if in_dygraph_mode(): infer_flags = list(1 for i in range(len(axes))) - inputs = {'Input': [input]} - if isinstance(starts, (list, tuple)): if utils._contain_var(starts): raise TypeError( @@ -9930,14 +9911,8 @@ def slice(input, axes, starts, ends): "The type of 'ends' in slice must be list[int] or tuple(int) in Dygraph mode, but " "received %s." % type(shape)) - attrs = { - 'axes': axes, - 'starts': starts, - 'ends': ends, - 'infer_flags': infer_flags - } - outs = core.ops.slice(inputs, attrs) - return outs['Out'][0] + return core.ops.slice(input, 'axes', axes, 'starts', starts, 'ends', + ends, 'infer_flags', infer_flags) if not isinstance(starts, (list, tuple, Variable)): raise ValueError( @@ -10392,6 +10367,14 @@ def scale(x, scale=1.0, bias=0.0, bias_after_scale=True, act=None, name=None): print(res) # [array([[ 3., 5., 7.], [ 9., 11., 13.]], dtype=float32)] """ + + if in_dygraph_mode(): + _scale = scale.numpy().item(0) if isinstance(scale, Variable) else scale + out = core.ops.scale(x, 'scale', + float(_scale), 'bias', + float(bias), 'bias_after_scale', bias_after_scale) + return dygraph_utils._append_activation_in_dygraph(out) + inputs = {'X': [x]} attrs = { 'bias': float(bias), @@ -10401,11 +10384,6 @@ def scale(x, scale=1.0, bias=0.0, bias_after_scale=True, act=None, name=None): inputs['ScaleTensor'] = [scale] else: attrs['scale'] = float(scale) - - if in_dygraph_mode(): - outs = core.ops.scale(inputs, attrs) - return dygraph_utils._append_activation_in_dygraph(outs['Out'][0]) - helper = LayerHelper('scale', **locals()) if name is None: out = helper.create_variable_for_type_inference(dtype=x.dtype) @@ -11369,9 +11347,7 @@ def mean(x, name=None): mean = fluid.layers.mean(input) """ if in_dygraph_mode(): - inputs = {"X": [x]} - outs = core.ops.mean(inputs) - return outs['Out'][0] + return core.ops.mean(x) helper = LayerHelper("mean", **locals()) check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], 'mean') @@ -11453,12 +11429,12 @@ def mul(x, y, x_num_col_dims=1, y_num_col_dims=1, name=None): """ - inputs = {"X": [x], "Y": [y]} - attrs = {"x_num_col_dims": x_num_col_dims, "y_num_col_dims": y_num_col_dims} if in_dygraph_mode(): - outs = core.ops.mul(inputs, attrs) - return outs['Out'][0] + return core.ops.mul(x, y, 'x_num_col_dims', x_num_col_dims, + 'y_num_col_dims', y_num_col_dims) + inputs = {"X": [x], "Y": [y]} + attrs = {"x_num_col_dims": x_num_col_dims, "y_num_col_dims": y_num_col_dims} helper = LayerHelper("mul", **locals()) check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], 'mul') check_variable_and_dtype(y, 'y', ['float16', 'float32', 'float64'], 'mul') diff --git a/python/paddle/fluid/layers/tensor.py b/python/paddle/fluid/layers/tensor.py index 20874338002..98ab713602c 100644 --- a/python/paddle/fluid/layers/tensor.py +++ b/python/paddle/fluid/layers/tensor.py @@ -255,15 +255,12 @@ def concat(input, axis=0, name=None): """ if in_dygraph_mode(): - inputs = {'X': input} if isinstance(axis, Variable): axis = axis.numpy() assert axis.shape == ( 1, ), "axis of type Variable should have shape [1]" axis = axis[0] - attrs = {'axis': axis} - outs = core.ops.concat(inputs, attrs) - return outs['Out'][0] + return core.ops.concat(input, 'axis', axis) if not isinstance(input, list): warnings.warn( @@ -586,12 +583,13 @@ def fill_constant(shape, dtype, value, force_cpu=False, out=None): shape)) else: shape = list(shape.numpy().astype(int)) - attrs['shape'] = shape + dtype = convert_np_dtype_to_dtype_(dtype) if out is None: out = _varbase_creator(dtype=dtype) - attrs['dtype'] = out.dtype - outputs = {'Out': [out]} - outs = core.ops.fill_constant({}, attrs, outputs) + core.ops.fill_constant(out, 'value', + float(value), 'force_cpu', force_cpu, 'dtype', + dtype, 'str_value', attrs['str_value'], 'shape', + shape) out.stop_gradient = True return out diff --git a/python/paddle/fluid/optimizer.py b/python/paddle/fluid/optimizer.py index ff6a42a4235..f46a52b6663 100644 --- a/python/paddle/fluid/optimizer.py +++ b/python/paddle/fluid/optimizer.py @@ -889,16 +889,11 @@ class SGDOptimizer(Optimizer): @no_grad def _append_optimize_op(self, block, param_and_grad): + lr = self._create_param_lr(param_and_grad) if framework.in_dygraph_mode(): - inputs = { - "Param": [param_and_grad[0]], - "Grad": [param_and_grad[1]], - "LearningRate": [self._create_param_lr(param_and_grad)] - } - attrs = {} - outputs = {'ParamOut': [param_and_grad[0]]} - outs = core.ops.sgd(inputs, attrs, outputs) - return outs['ParamOut'][0] + core.ops.sgd(param_and_grad[0], lr, param_and_grad[1], + param_and_grad[0]) + return None assert isinstance(block, framework.Block) # create the optimize op @@ -907,7 +902,7 @@ class SGDOptimizer(Optimizer): inputs={ "Param": param_and_grad[0], "Grad": param_and_grad[1], - "LearningRate": self._create_param_lr(param_and_grad) + "LearningRate": lr }, outputs={"ParamOut": param_and_grad[0]}, stop_gradient=True) @@ -1009,24 +1004,27 @@ class MomentumOptimizer(Optimizer): velocity_acc = self._get_accumulator(self._velocity_acc_str, param_and_grad[0]) - attrs = {"mu": self._momentum, "use_nesterov": self._use_nesterov} + lr = self._create_param_lr(param_and_grad) + if framework.in_dygraph_mode(): + _, _ = core.ops.momentum(param_and_grad[0], param_and_grad[1], + velocity_acc, lr, param_and_grad[0], + velocity_acc, 'mu', self._momentum, + 'use_nesterov', self._use_nesterov) + return None + + attrs = {"mu": self._momentum, "use_nesterov": self._use_nesterov} inputs = { "Param": [param_and_grad[0]], "Grad": [param_and_grad[1]], "Velocity": [velocity_acc], - "LearningRate": [self._create_param_lr(param_and_grad)] + "LearningRate": [lr] } outputs = { "ParamOut": [param_and_grad[0]], "VelocityOut": [velocity_acc] } - - if framework.in_dygraph_mode(): - core.ops.momentum(inputs, attrs, outputs) - return None - # create the momentum optimize op momentum_op = block.append_op( type=self.type, @@ -1849,12 +1847,27 @@ class AdamOptimizer(Optimizer): param_and_grad[0]) beta2_pow_acc = self._get_accumulator(self._beta2_pow_acc_str, param_and_grad[0]) - + lr = self._create_param_lr(param_and_grad) # create the adam optimize op + + if framework.in_dygraph_mode(): + _beta1 = self._beta1 if not isinstance( + self._beta1, Variable) else self._beta1.numpy().item(0) + _beta2 = self._beta2 if not isinstance( + self._beta2, Variable) else self._beta2.numpy().item(0) + _, _, _, _, _ = core.ops.adam( + param_and_grad[0], param_and_grad[1], lr, moment1, moment2, + beta1_pow_acc, beta2_pow_acc, param_and_grad[0], moment1, + moment2, beta1_pow_acc, beta2_pow_acc, 'epsilon', self._epsilon, + 'lazy_mode', self._lazy_mode, 'min_row_size_to_use_multithread', + 1000, 'beta1', _beta1, 'beta2', _beta2) + + return None + inputs = { "Param": [param_and_grad[0]], "Grad": [param_and_grad[1]], - "LearningRate": [self._create_param_lr(param_and_grad)], + "LearningRate": [lr], "Moment1": [moment1], "Moment2": [moment2], "Beta1Pow": [beta1_pow_acc], @@ -1882,10 +1895,6 @@ class AdamOptimizer(Optimizer): else: attrs['beta2'] = self._beta2 - if framework.in_dygraph_mode(): - core.ops.adam(inputs, attrs, outputs) - return None - adam_op = block.append_op( type=self.type, inputs=inputs, diff --git a/python/paddle/fluid/regularizer.py b/python/paddle/fluid/regularizer.py index 439d60ad674..96b4f16c961 100644 --- a/python/paddle/fluid/regularizer.py +++ b/python/paddle/fluid/regularizer.py @@ -54,7 +54,7 @@ def _create_regularization_of_grad(param, grad, regularization=None): inputs = {"X": [grad, regularization_term]} outputs = {"Out": [new_grad]} if in_dygraph_mode(): - core.ops.sum(inputs, {}, outputs) + new_grad = core.ops.sum([grad, regularization_term]) else: grad.block.append_op(type='sum', inputs=inputs, outputs=outputs) @@ -183,8 +183,7 @@ class L2DecayRegularizer(WeightDecayRegularizer): attrs = {"scale": self._regularization_coeff} if framework.in_dygraph_mode(): - outs = core.ops.scale(inputs, attrs) - return outs['Out'][0] + return core.ops.scale(param, "scale", self._regularization_coeff) else: decay = block.create_var( dtype=param.dtype, shape=param.shape, lod_level=param.lod_level) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_star_gan_with_gradient_penalty.py b/python/paddle/fluid/tests/unittests/test_imperative_star_gan_with_gradient_penalty.py index afb113ed941..176895aa9e7 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_star_gan_with_gradient_penalty.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_star_gan_with_gradient_penalty.py @@ -112,9 +112,9 @@ class InstanceNorm(fluid.dygraph.Layer): def forward(self, input): if fluid.in_dygraph_mode(): - inputs = {'X': [input], 'Scale': [self.scale], 'Bias': [self.bias]} - attrs = {'epsilon': self.epsilon} - return fluid.core.ops.instance_norm(inputs, attrs)['Y'][0] + out, _, _ = fluid.core.ops.instance_norm( + input, self.scale, self.bias, 'epsilon', self.epsilon) + return out else: return fluid.layers.instance_norm( input, diff --git a/python/paddle/fluid/tests/unittests/test_op_function_generator.py b/python/paddle/fluid/tests/unittests/test_op_function_generator.py index 72c74d23ae7..c59eca90d75 100644 --- a/python/paddle/fluid/tests/unittests/test_op_function_generator.py +++ b/python/paddle/fluid/tests/unittests/test_op_function_generator.py @@ -28,8 +28,7 @@ class TestTracedLayer(fluid.dygraph.Layer): super(TestTracedLayer, self).__init__(name_scope) def forward(self, input): - inputs = {'X': [input] if isinstance(input, fluid.Variable) else input} - return core.ops.relu(inputs)['Out'][0] + return core.ops.relu(input) class TestVariable(unittest.TestCase): @@ -47,9 +46,7 @@ class TestVariable(unittest.TestCase): x.stop_gradient = False res1 = layers.elementwise_add(x, y) - - inputs = {'X': [x], 'Y': [y]} - res2 = core.ops.elementwise_add(inputs)['Out'][0] + res2 = core.ops.elementwise_add(x, y) self.assertTrue(np.array_equal(res1.numpy(), res2.numpy())) @@ -61,9 +58,7 @@ class TestVariable(unittest.TestCase): y = fluid.dygraph.to_variable(b) res1 = layers.elementwise_mul(x, y) - - inputs = {'X': [x], 'Y': [y]} - res2 = core.ops.elementwise_mul(inputs)['Out'][0] + res2 = core.ops.elementwise_mul(x, y) self.assertTrue(np.array_equal(res1.numpy(), res2.numpy())) @@ -73,9 +68,7 @@ class TestVariable(unittest.TestCase): x = fluid.dygraph.to_variable(a) res1 = layers.relu(x) - - inputs = {'X': [x]} - res2 = core.ops.relu(inputs)['Out'][0] + res2 = core.ops.relu(x) self.assertTrue(np.array_equal(res1.numpy(), res2.numpy())) @@ -88,8 +81,7 @@ class TestVariable(unittest.TestCase): x.stop_gradient = False y.stop_gradient = False - inputs = {'X': [x], 'Y': [y]} - loss = core.ops.elementwise_mul(inputs)['Out'][0] + loss = core.ops.elementwise_mul(x, y) loss.backward() x_grad = x.gradient() @@ -104,7 +96,7 @@ class TestVariable(unittest.TestCase): a = np.random.uniform(-1, 1, self.shape).astype(self.dtype) x = fluid.dygraph.to_variable(a) res_dygraph, static_layer = TracedLayer.trace( - layer, inputs=[x]) # dygraph out + layer, inputs=x) # dygraph out res_static_graph = static_layer([x])[0] self.assertTrue( -- GitLab