diff --git a/mindspore/nn/wrap/loss_scale.py b/mindspore/nn/wrap/loss_scale.py index 4d929352b3fe347671712e1f3906f1f80b8d1937..1ce3179273caa4f85ecc4dc627f8f77de234667d 100644 --- a/mindspore/nn/wrap/loss_scale.py +++ b/mindspore/nn/wrap/loss_scale.py @@ -25,7 +25,6 @@ from ...ops import operations as P from ...ops.operations import NPUGetFloatStatus, NPUAllocFloatStatus, NPUClearFloatStatus, ReduceSum, LessEqual, \ ControlDepend from ...common import dtype as mstype -import mindspore.context as context _grad_scale = C.MultitypeFuncGraph("grad_scale") reciprocal = P.Reciprocal() @@ -35,12 +34,6 @@ reciprocal = P.Reciprocal() def tensor_grad_scale(scale, grad): return grad * F.cast(reciprocal(scale), F.dtype(grad)) -_grad_overflow = C.MultitypeFuncGraph("_grad_overflow") -grad_overflow = P.FloatStatus() - -@_grad_overflow.register("Tensor") -def _tensor_grad_overflow(grad): - return grad_overflow(grad) class DynamicLossScaleUpdateCell(Cell): r""" @@ -204,15 +197,9 @@ class TrainOneStepWithLossScaleCell(Cell): self.optimizer = optimizer self.grad = C.GradOperation('grad', get_by_list=True, sens_param=True) self.hyper_map = C.HyperMap() - if context.get_context("device_target") == "GPU": - self.gpu_target = True - self.float_status = P.FloatStatus() - self.addn = P.AddN() - else: - self.gpu_target = False - self.alloc_status = NPUAllocFloatStatus() - self.get_status = NPUGetFloatStatus() - self.clear_status = NPUClearFloatStatus() + self.alloc_status = NPUAllocFloatStatus() + self.get_status = NPUGetFloatStatus() + self.clear_status = NPUClearFloatStatus() self.reduce_sum = ReduceSum(keep_dims=False) self.base = Tensor(1, mstype.float32) self.less_equal = LessEqual() @@ -237,11 +224,10 @@ class TrainOneStepWithLossScaleCell(Cell): def construct(self, data, label, sens=None): weights = self.weights loss = self.network(data, label) - if not self.gpu_target: - # init overflow buffer - init = self.alloc_status() - # clear overflow buffer - self.clear_status(init) + # init overflow buffer + init = self.alloc_status() + # clear overflow buffer + self.clear_status(init) if sens is None: scaling_sens = self.loss_scale else: @@ -251,14 +237,10 @@ class TrainOneStepWithLossScaleCell(Cell): if self.reducer_flag: # apply grad reducer on grads grads = self.grad_reducer(grads) - if not self.gpu_target: - # get the overflow buffer - self.get_status(init) - # sum overflow buffer elements, 0:not overflow , >0:overflow - flag_sum = self.reduce_sum(init, (0,)) - else: - flag_sum = self.hyper_map(F.partial(_grad_overflow), grads) - flag_sum = self.addn(flag_sum) + # get the overflow buffer + self.get_status(init) + # sum overflow buffer elements, 0:not overflow , >0:overflow + flag_sum = self.reduce_sum(init, (0,)) if self.is_distributed: # sum overflow flag over devices flag_reduce = self.allreduce(flag_sum) diff --git a/mindspore/ops/operations/__init__.py b/mindspore/ops/operations/__init__.py index 48a985b33d18f2371c1dcdbbef157da65d4edb0b..37a3b38bb6796605db1144a316a18619e038ecc8 100644 --- a/mindspore/ops/operations/__init__.py +++ b/mindspore/ops/operations/__init__.py @@ -44,7 +44,7 @@ from .math_ops import (Abs, ACos, AddN, AssignAdd, AssignSub, Atan2, BatchMatMul LogicalNot, LogicalOr, MatMul, Maximum, Minimum, Mul, Neg, NMSWithMask, NotEqual, NPUAllocFloatStatus, NPUClearFloatStatus, - NPUGetFloatStatus, Pow, RealDiv, IsNan, IsInf, IsFinite, FloatStatus, + NPUGetFloatStatus, Pow, RealDiv, Reciprocal, CumSum, Sin, Sqrt, Rsqrt, Square, Sub, TensorAdd, Sign, Round) @@ -151,10 +151,6 @@ __all__ = [ 'Neg', 'Slice', 'DType', - 'IsNan', - 'IsInf', - 'IsFinite', - 'FloatStatus', 'NPUAllocFloatStatus', 'NPUGetFloatStatus', 'NPUClearFloatStatus', diff --git a/mindspore/ops/operations/math_ops.py b/mindspore/ops/operations/math_ops.py index 5f8c24d78b5cd40cd054f2dee41f6171b5fb8855..175b72560f67a44414214f7b41f5c185ff808919 100644 --- a/mindspore/ops/operations/math_ops.py +++ b/mindspore/ops/operations/math_ops.py @@ -1541,89 +1541,6 @@ class LogicalOr(_LogicBinaryOp): def infer_dtype(self, x_dtype, y_dtype): return _LogicBinaryOp.do_infer_dtype(x_dtype, y_dtype, (mstype.bool_,), self.prim_name()) -class IsNan(PrimitiveWithInfer): - """ - Judging which elements are nan for each position - Inputs: - - **input_x** (Tensor) - The input tensor. - - Outputs: - Tensor, has the same shape of input. - """ - - @prim_attr_register - def __init__(self): - """init IsNan""" - self.init_prim_io_names(inputs=['x'], outputs=['output']) - - def infer_shape(self, x_shape): - return x_shape - - def infer_dtype(self, x_dtype): - return mstype.bool_ - -class IsInf(PrimitiveWithInfer): - """ - Judging which elements are inf or -inf for each position - Inputs: - - **input_x** (Tensor) - The input tensor. - - Outputs: - Tensor, has the same shape of input. - """ - - @prim_attr_register - def __init__(self): - """init IsInf""" - self.init_prim_io_names(inputs=['x'], outputs=['output']) - - def infer_shape(self, x_shape): - return x_shape - - def infer_dtype(self, x_dtype): - return mstype.bool_ - -class IsFinite(PrimitiveWithInfer): - """ - Judging which elements are finite for each position - Inputs: - - **input_x** (Tensor) - The input tensor. - - Outputs: - Tensor, has the same shape of input. - """ - - @prim_attr_register - def __init__(self): - """init IsFinite""" - self.init_prim_io_names(inputs=['x'], outputs=['output']) - - def infer_shape(self, x_shape): - return x_shape - - def infer_dtype(self, x_dtype): - return mstype.bool_ - -class FloatStatus(PrimitiveWithInfer): - """ - Determine if the elements contains nan, inf or -inf - Inputs: - - **input_x** (Tensor) - The input tensor. - - Outputs: - Tensor, has the shape of `(1,)`. - """ - - @prim_attr_register - def __init__(self): - """init FloatStatus""" - self.init_prim_io_names(inputs=['x'], outputs=['output']) - - def infer_shape(self, x_shape): - return [1] - - def infer_dtype(self, x_dtype): - return x_dtype class NPUAllocFloatStatus(PrimitiveWithInfer): """