diff --git a/python/paddle/fluid/dygraph/dygraph_utils.py b/python/paddle/fluid/dygraph/dygraph_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..cb274789da0aaea38847d2ebad19b965f0bf2d11 --- /dev/null +++ b/python/paddle/fluid/dygraph/dygraph_utils.py @@ -0,0 +1,64 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .. import core +from ..framework import dygraph_only + + +@dygraph_only +def _append_activation_in_dygraph(input, + act=None, + use_cudnn=False, + use_mkldnn=False): + """Append activation in dygraph mode. + + Args: + input: the input variable. + act: activation type + use_mkldnn: if use mkldnn + use_cudnn: if use cudnn + + Return the Variable after append activation + """ + if not act: + return input + + attrs = {'use_cudnn': use_cudnn, 'use_mkldnn': use_mkldnn} + inputs = {"X": [input]} + act_op = getattr(core.ops, act) + res = act_op(inputs, attrs) + return res['Out'][0] + + +@dygraph_only +def _append_bias_in_dygraph( + input, + bias=None, + axis=1, ): + """Append bias operation in dygraph mode. + + Args: + input: the input variable. + bias: the bias to be appended + axis: the axis to perform operation + + Return the Variable after bias operation + """ + if not bias: + return input + + attrs = {'axis': axis} + inputs = {'X': [input], 'Y': [bias]} + outs = core.ops.elementwise_add(inputs, attrs) + return outs['Out'][0] diff --git a/python/paddle/fluid/dygraph/nn.py b/python/paddle/fluid/dygraph/nn.py index 38f2a30dc105937957e94b6b6d5e4c4c1837e48b..3292a7ee4eb1155a486ae6d7499d23b70a344871 100644 --- a/python/paddle/fluid/dygraph/nn.py +++ b/python/paddle/fluid/dygraph/nn.py @@ -15,9 +15,9 @@ from __future__ import print_function from six.moves import reduce - from .. import core from ..layers import utils +from ..dygraph import dygraph_utils from . import layers from ..framework import Variable, in_dygraph_mode, OpProtoHolder, Parameter, _dygraph_tracer_ from ..param_attr import ParamAttr @@ -235,6 +235,29 @@ class Conv2D(layers.Layer): self._bias_param = value def forward(self, input): + inputs = { + 'Input': [input], + 'Filter': [self._filter_param], + } + attrs = { + 'strides': self._stride, + 'paddings': self._padding, + 'dilations': self._dilation, + 'groups': self._groups if self._groups else 1, + 'use_cudnn': self._use_cudnn, + 'use_mkldnn': False, + } + + if in_dygraph_mode(): + outs = core.ops.conv2d(inputs, attrs) + pre_bias = outs['Output'][0] + + pre_act = dygraph_utils._append_bias_in_dygraph(pre_bias, + self._bias_param, 1) + + return dygraph_utils._append_activation_in_dygraph(pre_act, + self._act) + pre_bias = self._helper.create_variable_for_type_inference( dtype=self._dtype) @@ -245,14 +268,7 @@ class Conv2D(layers.Layer): 'Filter': self._filter_param, }, outputs={"Output": pre_bias}, - attrs={ - 'strides': self._stride, - 'paddings': self._padding, - 'dilations': self._dilation, - 'groups': self._groups if self._groups else 1, - 'use_cudnn': self._use_cudnn, - 'use_mkldnn': False, - }) + attrs=attrs) if self._bias_param is not None: pre_act = self._helper.create_variable_for_type_inference( @@ -858,23 +874,30 @@ class Pool2D(layers.Layer): self._l_type = 'pool2d' def forward(self, input): + attrs = { + "pooling_type": self._pool_type, + "ksize": self._pool_size, + "global_pooling": self._global_pooling, + "strides": self._pool_stride, + "paddings": self._pool_padding, + "use_cudnn": self._use_cudnn, + "ceil_mode": self._ceil_mode, + "use_mkldnn": False, + "exclusive": self._exclusive, + } + inputs = {"X": [input]} + + if in_dygraph_mode(): + outs = core.ops.pool2d(inputs, attrs) + return outs['Out'][0] + pool_out = self._helper.create_variable_for_type_inference(self._dtype) self._helper.append_op( type=self._l_type, inputs={"X": input}, outputs={"Out": pool_out}, - attrs={ - "pooling_type": self._pool_type, - "ksize": self._pool_size, - "global_pooling": self._global_pooling, - "strides": self._pool_stride, - "paddings": self._pool_padding, - "use_cudnn": self._use_cudnn, - "ceil_mode": self._ceil_mode, - "use_mkldnn": False, - "exclusive": self._exclusive, - }) + attrs=attrs) return pool_out @@ -948,17 +971,26 @@ class Linear(layers.Layer): shape=[output_dim], attr=bias_attr, dtype=dtype, is_bias=True) def forward(self, input): + attrs = { + "transpose_X": False, + "transpose_Y": False, + "alpha": 1, + } + inputs = {"X": [input], "Y": [self.weight]} + + if in_dygraph_mode(): + outs = core.ops.matmul(inputs, attrs) + pre_bias = outs['Out'][0] + + pre_act = dygraph_utils._append_bias_in_dygraph( + pre_bias, self.bias, axis=len(input.shape) - 1) + + return dygraph_utils._append_activation_in_dygraph(pre_act, + self._act) + tmp = self._helper.create_variable_for_type_inference(self._dtype) self._helper.append_op( - type="matmul", - inputs={"X": input, - "Y": self.weight}, - outputs={"Out": tmp}, - attrs={ - "transpose_X": False, - "transpose_Y": False, - "alpha": 1, - }) + type="matmul", inputs=inputs, outputs={"Out": tmp}, attrs=attrs) if self.bias: pre_activation = self._helper.create_variable_for_type_inference( dtype=self._dtype) @@ -1548,6 +1580,7 @@ class Embedding(layers.Layer): 'remote_prefetch': self._remote_prefetch, 'padding_idx': self._padding_idx } + if in_dygraph_mode(): inputs = {'Ids': [input], 'W': [self._w]} outs = core.ops.lookup_table_v2(inputs, attrs) diff --git a/python/paddle/fluid/layers/loss.py b/python/paddle/fluid/layers/loss.py index 9d9315c79b8a72039d6284d7be5f641294b0476b..9aaa6afd7e4e93d025b699e95b91a46a8003e037 100644 --- a/python/paddle/fluid/layers/loss.py +++ b/python/paddle/fluid/layers/loss.py @@ -234,36 +234,46 @@ def cross_entropy(input, label, soft_label=False, ignore_index=kIgnoreIndex): predict = fluid.layers.fc(input=x, size=class_num, act='softmax') cost = fluid.layers.cross_entropy(input=predict, label=label) """ + if not soft_label: + return cross_entropy2(input, label, ignore_index) + + inputs = {'X': [input], 'Label': [label]} + attrs = {"soft_label": soft_label, "ignore_index": ignore_index} + + if in_dygraph_mode(): + outs = core.ops.cross_entropy(inputs, attrs) + return outs['Y'][0] check_type_and_dtype(input, 'input', Variable, ['float16', 'float32', 'float64'], 'cross_entropy') - if not soft_label: - return cross_entropy2(input, label, ignore_index) helper = LayerHelper('cross_entropy', **locals()) out = helper.create_variable_for_type_inference(dtype=input.dtype) helper.append_op( - type='cross_entropy', - inputs={'X': [input], - 'Label': [label]}, - outputs={'Y': [out]}, - attrs={"soft_label": soft_label, - "ignore_index": ignore_index}) + type='cross_entropy', inputs=inputs, outputs={'Y': [out]}, attrs=attrs) return out def cross_entropy2(input, label, ignore_index=kIgnoreIndex): + inputs = {'X': [input], 'Label': [label]} + attrs = {'ignore_index': ignore_index} + + if in_dygraph_mode(): + outs = core.ops.cross_entropy2(inputs, attrs) + return outs['Y'][0] + + check_type_and_dtype(input, 'input', Variable, + ['float16', 'float32', 'float64'], 'cross_entropy2') helper = LayerHelper('cross_entropy2', **locals()) out = helper.create_variable_for_type_inference(dtype=input.dtype) xshape = helper.create_variable_for_type_inference(dtype=input.dtype) match_x = helper.create_variable_for_type_inference(dtype=input.dtype) helper.append_op( type='cross_entropy2', - inputs={'X': [input], - 'Label': [label]}, + inputs=inputs, outputs={'Y': [out], 'MatchX': [match_x], 'XShape': [xshape]}, - attrs={'ignore_index': ignore_index}) + attrs=attrs) return out diff --git a/python/paddle/fluid/layers/metric_op.py b/python/paddle/fluid/layers/metric_op.py index 4ffe65023063aeecce09a5908ff51522cdeca8e8..e19a8226ace1d7dd9fce29f6003392772e4e43cf 100644 --- a/python/paddle/fluid/layers/metric_op.py +++ b/python/paddle/fluid/layers/metric_op.py @@ -20,7 +20,8 @@ from __future__ import print_function import warnings from ..layer_helper import LayerHelper from ..initializer import Normal, Constant -from ..framework import Variable +from ..framework import Variable, in_dygraph_mode, _varbase_creator +from .. import core from ..param_attr import ParamAttr from . import nn from ..data_feeder import check_type_and_dtype @@ -71,6 +72,26 @@ def accuracy(input, label, k=1, correct=None, total=None): #[array([0.6666667], dtype=float32)] """ + if in_dygraph_mode(): + topk_out, topk_indices = nn.topk(input, k=k) + inputs = { + "Out": [topk_out], + "Indices": [topk_indices], + "Label": [label] + } + acc_out = _varbase_creator(dtype="float32") + if correct is None: + correct = _varbase_creator(dtype="int64") + if total is None: + total = _varbase_creator(dtype="int64") + outputs = { + "Accuracy": [acc_out], + "Correct": [correct], + "Total": [total] + } + outs = core.ops.accuracy(inputs, {}, outputs) + return outs['Accuracy'][0] + helper = LayerHelper("accuracy", **locals()) check_type_and_dtype(input, 'input', Variable, ['float16', 'float32', 'float64'], 'accuracy') diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 8c1d9f40882888a32d0f0a3ff446cd2895a699fb..f15eabe954382ea378da7724e3521792fb9f5f55 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -26,6 +26,7 @@ from ..layer_helper import LayerHelper from ..initializer import Normal, Constant, NumpyArrayInitializer from ..framework import Variable, OpProtoHolder, in_dygraph_mode, dygraph_only, _dygraph_tracer, default_main_program from ..dygraph import base +from ..dygraph import dygraph_utils from ..param_attr import ParamAttr from .layer_function_generator import autodoc, templatedoc, _generate_doc_string_ from .tensor import concat, assign, fill_constant, zeros @@ -186,28 +187,6 @@ __all__ = [ ] -@dygraph_only -def _append_activation_in_dygraph(input, - act=None, - use_cudnn=False, - use_mkldnn=False): - """Append activation in dygraph mode. - - Args: - input: the input variable. - act: activation type - use_mkldnn: if use mkldnn - use_cudnn: if use cudnn - - Return the Variable after append activation - """ - attrs = {'use_cudnn': use_cudnn, 'use_mkldnn': use_mkldnn} - inputs = {"X": [input]} - act_op = getattr(core.ops, act) - res = act_op(inputs, attrs) - return res['Out'][0] - - @dygraph_only def _elementwise_op_in_dygraph(x, y, @@ -219,13 +198,10 @@ def _elementwise_op_in_dygraph(x, inputs = {'X': [x], 'Y': [y]} op = getattr(core.ops, op_name) outs = op(inputs, attrs) - pre_act = outs['Out'][0] + out = outs['Out'][0] - if not act: - return pre_act - else: - return _append_activation_in_dygraph( - pre_act, act, use_mkldnn=use_mkldnn) + return dygraph_utils._append_activation_in_dygraph( + out, act, use_mkldnn=use_mkldnn) def fc(input, @@ -4736,15 +4712,23 @@ def topk(input, k, name=None): vk_values, vk_indices = layers.topk(input2, k=vk) #vk_values.shape=[None, 13, k], vk_indices.shape=[None, 13, k] """ - helper = LayerHelper("top_k", **locals()) - values = helper.create_variable_for_type_inference(dtype=input.dtype) - indices = helper.create_variable_for_type_inference(dtype="int64") inputs = {"X": [input]} - attrs = None + attrs = {} if isinstance(k, Variable): - inputs['K'] = k + inputs['K'] = [k] else: attrs = {'k': k} + + if in_dygraph_mode(): + outs = core.ops.top_k(inputs, attrs) + outs['Out'][0].stop_gradient = True + outs['Indices'][0].stop_gradient = True + return outs['Out'][0], outs['Indices'][0] + + helper = LayerHelper("top_k", **locals()) + values = helper.create_variable_for_type_inference(dtype=input.dtype) + indices = helper.create_variable_for_type_inference(dtype="int64") + helper.append_op( type="top_k", inputs=inputs, @@ -5594,11 +5578,8 @@ def reshape(x, shape, actual_shape=None, act=None, inplace=False, name=None): inputs = {'X': [x]} outs = core.ops.reshape2(inputs, attrs) - pre_act = outs['Out'][0] - if act is None: - return pre_act - else: - return _append_activation_in_dygraph(pre_act, act) + out = outs['Out'][0] + return dygraph_utils._append_activation_in_dygraph(out, act) check_type_and_dtype(x, 'x', Variable, ['float16', 'float32', 'float64', 'int32', 'int64'], @@ -11333,6 +11314,10 @@ def mean(x, name=None): name='data', shape=[2, 3], dtype='float32') mean = fluid.layers.mean(input) """ + if in_dygraph_mode(): + inputs = {"X": [x]} + outs = core.ops.mean(inputs) + return outs['Out'][0] helper = LayerHelper("mean", **locals()) check_type_and_dtype(x, 'x', Variable, ['float16', 'float32', 'float64'], diff --git a/python/paddle/fluid/optimizer.py b/python/paddle/fluid/optimizer.py index 487dc677d12afc2ba51b77367cf8f160b6ea81e6..5b1a67321061a6f2a132d723934ea2c3b89588f1 100644 --- a/python/paddle/fluid/optimizer.py +++ b/python/paddle/fluid/optimizer.py @@ -1712,20 +1712,20 @@ class AdamOptimizer(Optimizer): # create the adam optimize op inputs = { - "Param": param_and_grad[0], - "Grad": param_and_grad[1], - "LearningRate": self._create_param_lr(param_and_grad), - "Moment1": moment1, - "Moment2": moment2, - "Beta1Pow": beta1_pow_acc, - "Beta2Pow": beta2_pow_acc + "Param": [param_and_grad[0]], + "Grad": [param_and_grad[1]], + "LearningRate": [self._create_param_lr(param_and_grad)], + "Moment1": [moment1], + "Moment2": [moment2], + "Beta1Pow": [beta1_pow_acc], + "Beta2Pow": [beta2_pow_acc] } outputs = { - "ParamOut": param_and_grad[0], - "Moment1Out": moment1, - "Moment2Out": moment2, - "Beta1PowOut": beta1_pow_acc, - "Beta2PowOut": beta2_pow_acc, + "ParamOut": [param_and_grad[0]], + "Moment1Out": [moment1], + "Moment2Out": [moment2], + "Beta1PowOut": [beta1_pow_acc], + "Beta2PowOut": [beta2_pow_acc], } attrs = { "epsilon": self._epsilon, @@ -1742,6 +1742,10 @@ class AdamOptimizer(Optimizer): else: attrs['beta2'] = self._beta2 + if framework.in_dygraph_mode(): + core.ops.adam(inputs, attrs, outputs) + return None + adam_op = block.append_op( type=self.type, inputs=inputs, diff --git a/python/paddle/fluid/tests/unittests/test_imperative_resnet.py b/python/paddle/fluid/tests/unittests/test_imperative_resnet.py index b8c242d920899bc35118433266e2f469d5e1b247..4fd7fec0e4acf55c7f9963e07c8b4e039d760d2b 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_resnet.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_resnet.py @@ -284,7 +284,7 @@ class TestDygraphResnet(unittest.TestCase): if traced_layer is not None: resnet.eval() traced_layer._switch(is_test=True) - out_dygraph = resnet([img]) + out_dygraph = resnet(img) out_static = traced_layer([img]) traced_layer._switch(is_test=False) helper.assertEachVar(out_dygraph, out_static) diff --git a/python/paddle/fluid/tests/unittests/test_layers.py b/python/paddle/fluid/tests/unittests/test_layers.py index c8419540cb91d2c265bdc685781f15758abf2314..c58ba9e387321b4618d98d0bad5e66f4d7f4a30b 100644 --- a/python/paddle/fluid/tests/unittests/test_layers.py +++ b/python/paddle/fluid/tests/unittests/test_layers.py @@ -1627,6 +1627,34 @@ class TestLayer(LayerTest): self.assertIsNotNone(out2) self.assertIsNotNone(out3) + def test_accuracy(self): + x = np.random.rand(3, 32, 32).astype("float32") + y = np.array([[1], [0], [1]]) + with self.static_graph(): + data = fluid.data(name="input", shape=[-1, 32, 32], dtype="float32") + label = fluid.data(name="label", shape=[-1, 1], dtype="int") + fc_out = fluid.layers.fc(input=data, size=10) + predict = fluid.layers.softmax(input=fc_out) + result = fluid.layers.accuracy(input=predict, label=label, k=5) + place = fluid.CPUPlace() + exe = fluid.Executor(place) + + exe.run(fluid.default_startup_program()) + x = np.random.rand(3, 32, 32).astype("float32") + y = np.array([[1], [0], [1]]) + static_out = exe.run(feed={"input": x, + "label": y}, + fetch_list=result[0]) + + with self.dynamic_graph(): + data = base.to_variable(x) + label = base.to_variable(y) + fc_out = fluid.layers.fc(data, size=10) + predict = fluid.layers.softmax(fc_out) + dynamic_out = fluid.layers.accuracy(input=predict, label=label, k=5) + + self.assertTrue(np.array_equal(static_out[0], dynamic_out.numpy())) + class TestBook(LayerTest): def test_all_layers(self):