!1836 update register info of BiasAddGrad and modify adam...

!1836 update register info of BiasAddGrad and modify adam optimizer&softmax_grad to match fusion rules
Merge pull request !1836 from shibeiji/r0.3

mindspore/nn/layer/basic.py

@@ -22,6 +22,7 @@ from mindspore.ops import operations as P
 from mindspore.ops import functional as F
 from mindspore.ops.functional import identity
 from mindspore.ops.operations import _inner_ops as inner
+from mindspore.ops.primitive import constexpr
 from mindspore.common.parameter import Parameter
 from mindspore._extends import cell_attr_register
 from mindspore.common.api import ms_function
@@ -236,6 +237,13 @@ class Dense(Cell):
         return str_info
 
 
+@constexpr
+def _is_equal_one(x):
+    if x is None:
+        return False
+    return bool(x.asnumpy().mean() == 1.0)
+
+
 class ClipByNorm(Cell):
     r"""
     Clips tensor values to a maximum :math:`L_2`-norm.
@@ -290,7 +298,10 @@ class ClipByNorm(Cell):
         l2sum_safe = self.select_(cond, l2sum, self.cast(ones_, self.dtype(l2sum)))
         l2norm = self.select_(cond, self.sqrt(l2sum_safe), l2sum)
 
-        intermediate = x * clip_norm
+        if _is_equal_one(clip_norm):
+            intermediate = x
+        else:
+            intermediate = x * clip_norm
         max_norm = self.max_op(l2norm, clip_norm)
         values_clip = self.cast(intermediate, mstype.float32) / self.expand_dims(max_norm, -1)
         values_clip = self.reshape(values_clip, self.shape(x))

mindspore/nn/optim/adam.py

@@ -67,9 +67,9 @@ def _update_run_op(beta1, beta2, eps, lr, weight_decay_tensor, param, m, v, grad
     next_v = op_mul(beta2, v_fp32) + op_mul(op_cast(F.tuple_to_array((1.0,)), mstype.float32)
                                             - beta2, op_square(gradient_fp32))
 
-    update = next_m / (op_sqrt(next_v) + eps)
+    update = next_m / (eps + op_sqrt(next_v))
     if decay_flag:
-        update = update + op_mul(weight_decay_tensor, param_fp32)
+        update = op_mul(weight_decay_tensor, param_fp32) + update
 
     update_with_lr = op_mul(lr, update)
     next_param = param_fp32 - op_reshape(update_with_lr, op_shape(param_fp32))

mindspore/ops/_grad/grad_nn_ops.py

@@ -309,7 +309,7 @@ def get_bprop_softmax(self):
     axis = self.axis
 
     def bprop(x, out, dout):
-        dx = mul(sub(dout, sum_func(mul(dout, out), axis)), out)
+        dx = mul(out, sub(dout, sum_func(mul(out, dout), axis)))
         return (dx,)
 
     return bprop

mindspore/ops/_op_impl/tbe/bias_add_grad.py

@@ -26,6 +26,8 @@ bias_add_grad_op_info = TBERegOp("BiasAddGrad") \
     .attr("data_format", "required", "str", "all") \
     .input(0, "output_backprop", False, "required", "all") \
     .output(0, "output", False, "required", "all") \
+    .dtype_format(DataType.F16_Default, DataType.F16_Default) \
+    .dtype_format(DataType.F16_FracNZ, DataType.F16_Default) \
     .dtype_format(DataType.F32_Default, DataType.F32_Default) \
     .dtype_format(DataType.F32_FracNZ, DataType.F32_Default) \
     .get_op_info()