Fix relu python call (#45082)

* add python final state

* fix bug

* fix bugs

* fix bug

* fix bug

* revert impl, final state mul not support selected rows

* fix softmax use cudnn error

* add softlable false unitest

* revert loss.py

paddle/phi/api/ext/tensor_compat.h

@@ -71,7 +71,7 @@ using experimental::gather_nd;
 using experimental::gelu;
 using experimental::gumbel_softmax;
 using experimental::imag;
-using experimental::increment;
+using experimental::increment_;
 using experimental::index_sample;
 using experimental::is_empty;
 using experimental::isclose;

paddle/phi/api/yaml/legacy_api.yaml

@@ -1246,6 +1246,7 @@
     func : IncrementInferMeta
   kernel :
     func : increment
+  inplace : (x -> out)
 
 - api : index_sample
   args : (Tensor x, Tensor index)

python/paddle/fluid/clip.py

@@ -267,7 +267,7 @@ class ClipGradByValue(ClipGradBase):
             if getattr(p, 'need_clip', True) is False:
                 params_and_grads.append((p, g))
                 continue
-            new_grad = layers.clip(x=g, min=self.min, max=self.max)
+            new_grad = paddle.clip(x=g, min=self.min, max=self.max)
             params_and_grads.append((p, new_grad))
         return params_and_grads
 

python/paddle/fluid/dygraph_utils.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 from . import core
-from .framework import dygraph_only
+from .framework import dygraph_only, in_dygraph_mode
 from paddle import _C_ops
 
 

python/paddle/fluid/layers/control_flow.py

@@ -1554,6 +1554,9 @@ def increment(x, value=1.0, in_place=True):
           counter = fluid.layers.zeros(shape=[1], dtype='float32') # [0.]
           fluid.layers.increment(counter) # [1.]
     """
+    if in_dygraph_mode():
+        return _C_ops.final_state_increment_(x, value)
+
     check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'],
                              'increment')
     helper = LayerHelper("increment", **locals())
@@ -1973,6 +1976,10 @@ def equal(x, y, cond=None, name=None):
           out1 = fluid.layers.equal(x=label,y=limit) #out1=[True, False]
           out2 = fluid.layers.equal(x=label_cond,y=limit, cond=out_cond) #out2=[False, True] out_cond=[False, True]
     """
+    if in_dygraph_mode():
+        default_axis = -1
+        return _C_ops.final_state_equal(x, y, default_axis)
+
     check_variable_and_dtype(x, "x", ["float32", "float64", "int32", "int64"],
                              "equal")
     check_variable_and_dtype(y, "y", ["float32", "float64", "int32", "int64"],

python/paddle/fluid/layers/nn.py

@@ -1454,6 +1454,9 @@ def softmax(input, use_cudnn=True, name=None, axis=-1):
 
     """
 
+    if in_dygraph_mode():
+        return _C_ops.final_state_softmax(input, axis)
+
     if _non_static_mode():
         return _C_ops.softmax(input, 'axis', axis, 'use_cudnn', use_cudnn)
 

python/paddle/fluid/layers/tensor.py

@@ -329,8 +329,7 @@ def concat(input, axis=0, name=None):
             axis = axis.item(0)
         if not isinstance(input, Variable):
             input = [t for t in input if t.shape.count(0) == 0]
-        out = _varbase_creator()
-        _C_ops.concat(input, out, 'axis', axis)
+        out = _C_ops.final_state_concat(input, axis)
         return out
 
     if _in_legacy_dygraph():

python/paddle/fluid/regularizer.py

@@ -16,7 +16,7 @@ from __future__ import print_function
 import logging
 
 from . import framework
-from .framework import _non_static_mode, _varbase_creator
+from .framework import _non_static_mode, _varbase_creator, in_dygraph_mode
 from . import core
 from paddle import _C_ops
 
@@ -252,6 +252,10 @@ class L1DecayRegularizer(WeightDecayRegularizer):
             decay = block.create_var(dtype=param.dtype,
                                      shape=param.shape,
                                      lod_level=param.lod_level)
+        if in_dygraph_mode():
+            sign = _C_ops.final_state_sign(param)
+            return _C_ops.final_state_scale(sign, self._regularization_coeff,
+                                            0.0, True)
 
         # Append sign op
         block.append_op(type='sign', inputs={"X": param}, outputs={"Out": sign})

python/paddle/fluid/tests/unittests/dygraph_to_static/test_se_resnet.py

@@ -316,7 +316,7 @@ class SeResNeXt(fluid.dygraph.Layer):
         y = fluid.layers.reshape(y, shape=[-1, self.pool2d_avg_output])
         out = self.out(y)
 
-        softmax_out = fluid.layers.softmax(out, use_cudnn=False)
+        softmax_out = fluid.layers.softmax(out)
         loss = fluid.layers.cross_entropy(input=softmax_out, label=label)
         avg_loss = paddle.mean(x=loss)
 

python/paddle/fluid/tests/unittests/test_cross_entropy_loss.py

@@ -560,8 +560,10 @@ class CrossEntropyLoss(unittest.TestCase):
 
     ###soft_label test 5
     def test_cross_entropy_loss_soft_2d(self):
+
+        def inner_cross_entropy_loss_soft_2d(soft_label):
             self.numeric_stable_mode = False
-        self.soft_label = True
+            self.soft_label = soft_label
             self.dtype = 'float32' if fluid.core.is_compiled_with_rocm(
             ) else 'float64'
             self.axis = -1
@@ -577,9 +579,11 @@ class CrossEntropyLoss(unittest.TestCase):
             self.logits = getattr(
                 self, "logits",
                 np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype))
-        softmax = np.apply_along_axis(stable_softmax, self.axis, self.logits)
+            softmax = np.apply_along_axis(stable_softmax, self.axis,
+                                          self.logits)
 
-        self.labels = np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype)
+            self.labels = np.random.uniform(0.1, 1.0,
+                                            self.shape).astype(self.dtype)
             self.labels /= np.sum(self.labels, axis=self.axis, keepdims=True)
 
             #1. numpy
@@ -638,6 +642,9 @@ class CrossEntropyLoss(unittest.TestCase):
             np.testing.assert_allclose(static_ret[0], expected, rtol=1e-05)
             np.testing.assert_allclose(dy_ret_value, expected, rtol=1e-05)
 
+        inner_cross_entropy_loss_soft_2d(True)
+        inner_cross_entropy_loss_soft_2d(False)
+
     ###soft_label test 6
     def test_cross_entropy_loss_soft_2d_weight_mean(self):
         self.numeric_stable_mode = False

python/paddle/fluid/tests/unittests/test_dygraph_multi_forward.py

@@ -214,4 +214,5 @@ class TestDygraphMultiForward(unittest.TestCase):
 
 
 if __name__ == '__main__':
+    paddle.enable_static()
     unittest.main()

python/paddle/fluid/tests/unittests/test_l1_loss.py

@@ -199,4 +199,5 @@ class TestClassL1Loss(unittest.TestCase):
 
 
 if __name__ == "__main__":
+    paddle.enable_static()
     unittest.main()

python/paddle/nn/functional/activation.py

@@ -231,7 +231,10 @@ def hardshrink(x, threshold=0.5, name=None):
             out = F.hardshrink(x) # [-1., 0., 2.5]
 
     """
-    if in_dynamic_mode():
+    if in_dygraph_mode():
+        return _C_ops.final_state_hard_shrink(x, threshold)
+
+    if _in_legacy_dygraph():
         return _C_ops.hard_shrink(x, 'threshold', threshold)
 
     check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'],
@@ -281,7 +284,10 @@ def hardtanh(x, min=-1.0, max=1.0, name=None):
             out = F.hardtanh(x) # [-1., 0.3, 1.]
     """
 
-    if in_dynamic_mode():
+    if in_dygraph_mode():
+        return _C_ops.final_state_brelu(x, min, max)
+
+    if _in_legacy_dygraph():
         return _C_ops.brelu(x, 't_min', min, 't_max', max)
 
     check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'],
@@ -337,7 +343,10 @@ def hardsigmoid(x, slope=0.1666667, offset=0.5, name=None):
             out = F.hardsigmoid(x) # [0., 1., 0.666667]
     """
 
-    if in_dynamic_mode():
+    if in_dygraph_mode():
+        return _C_ops.final_state_hard_sigmoid(x, slope, offset)
+
+    if _in_legacy_dygraph():
         return _C_ops.hard_sigmoid(x, 'slope', slope, 'offset', offset)
 
     check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'],
@@ -744,7 +753,10 @@ def log_sigmoid(x, name=None):
             out = F.log_sigmoid(x) # [-0.313262 -0.126928 -0.0485874 -0.0181499]
     """
 
-    if in_dynamic_mode():
+    if in_dygraph_mode():
+        return _C_ops.final_state_logsigmoid(x)
+
+    if _in_legacy_dygraph():
         return _C_ops.logsigmoid(x)
 
     check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'],
@@ -1099,7 +1111,7 @@ def softmax(x, axis=-1, dtype=None, name=None):
 
     if in_dygraph_mode():
         outs_cast = x if dtype is None \
-            else _C_ops.cast(x, 'in_dtype', x.dtype, 'out_dtype', dtype)
+            else _C_ops.final_state_cast(x, dtype)
         return _C_ops.final_state_softmax(outs_cast, axis)
 
     if _in_legacy_dygraph():
@@ -1413,7 +1425,10 @@ def tanhshrink(x, name=None):
             x = paddle.to_tensor(np.array([-0.4, -0.2, 0.1, 0.3]))
             out = F.tanhshrink(x) # [-0.020051, -0.00262468, 0.000332005, 0.00868739]
     """
-    if in_dynamic_mode():
+    if in_dygraph_mode():
+        return _C_ops.final_state_tanh_shrink(x)
+
+    if _in_legacy_dygraph():
         return _C_ops.tanh_shrink(x)
 
     check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'],
@@ -1459,7 +1474,10 @@ def thresholded_relu(x, threshold=1.0, name=None):
             out = F.thresholded_relu(x) # [2., 0., 0.]
     """
 
-    if in_dynamic_mode():
+    if in_dygraph_mode():
+        return _C_ops.final_state_thresholded_relu(x, threshold)
+
+    if _in_legacy_dygraph():
         return _C_ops.thresholded_relu(x, 'threshold', threshold)
 
     check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'],
@@ -1532,12 +1550,15 @@ def log_softmax(x, axis=-1, dtype=None, name=None):
     if (dtype is not None) and (not isinstance(dtype, core.VarDesc.VarType)):
         dtype = convert_np_dtype_to_dtype_(dtype)
 
-    if _non_static_mode():
+    if in_dygraph_mode():
         if dtype is not None:
-            x = _C_ops.cast(x, 'in_dtype', x.dtype, 'out_dtype', dtype)
+            x = _C_ops.final_state_cast(x, dtype)
+        return _C_ops.final_state_log_softmax(x, axis)
+
     if _in_legacy_dygraph():
+        if dtype is not None:
+            x = _C_ops.cast(x, 'in_dtype', x.dtype, 'out_dtype', dtype)
         return _C_ops.log_softmax(x, 'axis', axis)
-        return _C_ops.final_state_log_softmax(x, axis)
 
     if dtype is None:
         check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'],

python/paddle/nn/functional/vision.py

@@ -21,7 +21,7 @@ import numpy as np
 from paddle import _C_ops
 from ...device import is_compiled_with_rocm
 from paddle import in_dynamic_mode
-from paddle.fluid.framework import in_dygraph_mode
+from paddle.fluid.framework import in_dygraph_mode, _in_legacy_dygraph
 from paddle.framework import _non_static_mode
 
 __all__ = []
@@ -335,8 +335,10 @@ def pixel_shuffle(x, upscale_factor, data_format="NCHW", name=None):
         raise ValueError(
             "Attr(data_format) should be 'NCHW' or 'NHWC'."
             "But recevie Attr(data_format): {} ".format(data_format))
+    if in_dygraph_mode():
+        return _C_ops.final_state_pixel_shuffle(x, upscale_factor, data_format)
 
-    if in_dynamic_mode():
+    if _in_legacy_dygraph():
         return _C_ops.pixel_shuffle(x, "upscale_factor", upscale_factor,
                                     "data_format", data_format)
 

python/paddle/tensor/math.py

@@ -3522,7 +3522,7 @@ def increment(x, value=1.0, name=None):
 
     """
     if in_dygraph_mode():
-        return _C_ops.final_state_increment( x, value)
+        return _C_ops.final_state_increment_( x, value)
 
     if _in_legacy_dygraph():
         return _C_ops.increment(x, 'step', value)

python/paddle/tensor/random.py

@@ -114,6 +114,8 @@ def poisson(x, name=None):
             # [5., 1., 3.]]
 
     """
+    if in_dygraph_mode():
+        return _C_ops.final_state_poisson(x)
 
     if paddle.in_dynamic_mode():
         return _C_ops.poisson(x)