diff --git a/mindspore/nn/layer/math.py b/mindspore/nn/layer/math.py index 8a714c1cdebb4472155501086687d97032e980c3..acd18d34deea34c157cd62f52d26392c583ddcbe 100644 --- a/mindspore/nn/layer/math.py +++ b/mindspore/nn/layer/math.py @@ -79,8 +79,8 @@ class Range(Cell): start (Union[int, float]): If `limit` is `None`, the value acts as limit in the range and first entry defaults to `0`. Otherwise, it acts as first entry in the range. limit (Union[int, float]): Acts as upper limit of sequence. If `None`, defaults to the value of `start` - while set the first entry of the range to `0`. - delta (Union[int, float]): Increment of the range. Default: 1. + while set the first entry of the range to `0`. It can not be equal to `start`. + delta (Union[int, float]): Increment of the range. It can not be equal to zero. Default: 1. Outputs: Tensor, the dtype is int if the dtype of `start`, `limit` and `delta` all are int. Otherwise, dtype is float. @@ -93,10 +93,12 @@ class Range(Cell): def __init__(self, start, limit=None, delta=1): super(Range, self).__init__() - validator.check_value_type("start", start, [int, float], None) - validator.check_value_type("delta", delta, [int, float], None) + validator.check_value_type("start", start, [int, float], self.cls_name) + validator.check_value_type("delta", delta, [int, float], self.cls_name) + if delta == 0: + raise ValueError("The input of `delta` can not be equal to zero.") if limit is not None: - validator.check_value_type("limit", limit, [int, float], None) + validator.check_value_type("limit", limit, [int, float], self.cls_name) if isinstance(start, int) and isinstance(limit, int) and isinstance(delta, int): self.dtype = mstype.int32 else: diff --git a/mindspore/nn/optim/adam.py b/mindspore/nn/optim/adam.py index ee5970cc7644854b82300c89f3ab1032434190eb..b475f80a759af7f86216823f5708e4ff7b4e9bed 100755 --- a/mindspore/nn/optim/adam.py +++ b/mindspore/nn/optim/adam.py @@ -265,14 +265,15 @@ class AdamWeightDecay(Optimizer): take the i-th value as the learning rate. When the learning_rate is float or learning_rate is a Tensor but the dims of the Tensor is 0, use fixed learning rate. - Other cases are not supported. Default: 1e-3. + Other cases are not supported. It should be equal to or + greater than 0. Default: 1e-3. beta1 (float): The exponential decay rate for the 1st moment estimates. Default: 0.9. Should be in range (0.0, 1.0). beta2 (float): The exponential decay rate for the 2nd moment estimates. Default: 0.999. Should be in range (0.0, 1.0). eps (float): Term added to the denominator to improve numerical stability. Default: 1e-6. Should be greater than 0. - weight_decay (float): Weight decay (L2 penalty). Default: 0.0. + weight_decay (float): Weight decay (L2 penalty). It should be equal to or greater than 0. Default: 0.0. decay_filter (Function): A function to determine whether to apply weight decay on parameters. Default: lambda x: 'LayerNorm' not in x.name and 'bias' not in x.name. @@ -322,18 +323,20 @@ class AdamWeightDecayDynamicLR(Optimizer): Args: params (list[Parameter]): A list of parameter, which will be updated. The element in `params` should be class mindspore.Parameter. - decay_steps (int): The steps of the decay. + decay_steps (int): The steps of the decay. It must be int and positive. warmup_steps (int): The steps of lr warm up. Default: 0. - learning_rate (float): A floating point value for the learning rate. Default: 0.001. - end_learning_rate (float): A floating point value for the end learning rate. Default: 0.0001. - power (float): The Power of the polynomial. Default: 10.0. + learning_rate (float): A floating point value for the learning rate. It should be equal to or + greater than 0. Default: 0.001. + end_learning_rate (float): A floating point value for the end learning rate. It should be equal + to or greater than 0. Default: 0.0001. + power (float): The Power of the polynomial. It must be positive. Default: 10.0. beta1 (float): The exponential decay rate for the 1st moment estimates. Default: 0.9. Should be in range (0.0, 1.0). beta2 (float): The exponential decay rate for the 2nd moment estimates. Default: 0.999. Should be in range (0.0, 1.0). eps (float): Term added to the denominator to improve numerical stability. Default: 1e-6. Should be greater than 0. - weight_decay (float): Weight decay (L2 penalty). Default: 0.0. + weight_decay (float): Weight decay (L2 penalty). It should be equal to or greater than 0. Default: 0.0. decay_filter (Function): A function to determine whether to apply weight decay on parameters. Default: lambda x: 'LayerNorm' not in x.name and 'bias' not in x.name. diff --git a/mindspore/nn/optim/optimizer.py b/mindspore/nn/optim/optimizer.py index 82a276404daef9f8f18a39dffc940857fc046413..4742562524e84338dd152e2677c5294e50827519 100755 --- a/mindspore/nn/optim/optimizer.py +++ b/mindspore/nn/optim/optimizer.py @@ -55,8 +55,8 @@ class Optimizer(Cell): take the i-th value as the learning rate. When the learning_rate is float or learning_rate is a Tensor but the dims of the Tensor is 0, use fixed learning rate. - Other cases are not supported. Should be greater than 0. - If the type of `learning_rate` input is int, it will be + Other cases are not supported. It should be equal to or greater + than 0. If the type of `learning_rate` input is int, it will be converted to float. parameters (Union[list[Parameter], list[dict]]): When the `parameters` is a list of `Parameter` which will be updated, the element in `parameters` should be class `Parameter`. When the `parameters` is a list of `dict`, diff --git a/mindspore/ops/operations/array_ops.py b/mindspore/ops/operations/array_ops.py index 6b1794c243964ad706952d32c5fb8086c37e9ce1..054e9dc8198c5e4cf8e83c0009590a7d8a298b35 100644 --- a/mindspore/ops/operations/array_ops.py +++ b/mindspore/ops/operations/array_ops.py @@ -537,8 +537,8 @@ class Range(PrimitiveWithInfer): start (float): If `limit` is `None`, the value acts as limit in the range and first entry defaults to `0`. Otherwise, it acts as first entry in the range. limit (float): Acts as upper limit of sequence. If `None`, defaults to the value of `start` - while set the first entry of the range to `0`. - delta (float): Increment of the range. Default: 1.0. + while set the first entry of the range to `0`. It can not be equal to `start`. + delta (float): Increment of the range. It can not be equal to zero. Default: 1.0. Inputs: - **input_x** (Tensor) - The assistant data. A `1-D` tensor of type float32 or int32. @@ -565,6 +565,15 @@ class Range(PrimitiveWithInfer): self.add_prim_attr("limit", self.limit) else: validator.check_value_type("limit", limit, [float], self.name) + validator.check('start', self.start, 'limit', self.limit, Rel.NE, self.name) + if self.delta == 0.0: + raise ValueError("The input of `delta` can not be equal to zero.") + if self.delta > 0.0 and self.start > self.limit: + raise ValueError(f"Limit should be greater than start when delta:{self.delta} is more than zero, " + f"but got start:{self.start}, limit:{self.limit}") + if self.delta < 0.0 and self.start < self.limit: + raise ValueError(f"Start should be greater than limit when delta:{self.delta} is less than zero, " + f"but got start:{self.start}, limit:{self.limit}") def infer_shape(self, x_shape): return x_shape diff --git a/mindspore/ops/operations/nn_ops.py b/mindspore/ops/operations/nn_ops.py index 39876cabcac676b1ac6fa7482b07c5fc668d0d6a..fb627b5b736d35acc2a8482818bef4277b944699 100644 --- a/mindspore/ops/operations/nn_ops.py +++ b/mindspore/ops/operations/nn_ops.py @@ -2829,8 +2829,7 @@ class ApplyProximalAdagrad(PrimitiveWithInfer): Inputs: - **var** (Tensor) - Variable to be updated. - **accum** (Tensor) - Accum to be updated. The shape must be the same as `var`'s shape. - - **lr** (Union[Number, Tensor]): The learning rate value, must be positive. It should be - a scalar tensor or number. + - **lr** (Union[Number, Tensor]): The learning rate value. It should be a scalar tensor or number. - **l1** (Union[Number, Tensor]): l1 regularization strength, must be greater than or equal to zero. It should be a scalar tensor or number. - **l2** (Union[Number, Tensor]): l2 regularization strength, must be greater than or equal to zero. @@ -2888,8 +2887,7 @@ class SparseApplyProximalAdagrad(PrimitiveWithInfer): Inputs: - **var** (Tensor) - Variable tensor to be updated. - **accum** (Tensor) - Variable tensor to be updated. The shape must be the same as `var`'s shape. - - **lr** (Union[Number, Tensor]): The learning rate value, must be positive. It should be - a scalar tensor or number. + - **lr** (Union[Number, Tensor]): The learning rate value. It should be a scalar tensor or number. - **l1** (Union[Number, Tensor]): l1 regularization strength, must be greater than or equal to zero. It should be a scalar tensor or number. - **l2** (Union[Number, Tensor]): l2 regularization strength, must be greater than or equal to zero. @@ -3124,17 +3122,17 @@ class SparseApplyFtrl(PrimitiveWithInfer): >>> def __init__(self): >>> super(SparseApplyFtrlNet, self).__init__() >>> self.sparse_apply_ftrl = P.SparseApplyFtrl(lr=0.01, l1=0.0, l2=0.0, lr_power=-0.5) - >>> self.var = Parameter(Tensor(np.random.random(3, 3).astype(np.float32)), name="var") - >>> self.accum = Parameter(Tensor(np.random.random(3, 3).astype(np.float32)), name="accum") - >>> self.linear = Parameter(Tensor(np.random.random(3, 3).astype(np.float32)), name="linear") + >>> self.var = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="var") + >>> self.accum = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="accum") + >>> self.linear = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="linear") >>> >>> def construct(self, grad, indices): - >>> out = self.apply_ftrl(self.var, self.accum, self.linear, grad, indices) + >>> out = self.sparse_apply_ftrl(self.var, self.accum, self.linear, grad, indices) >>> return out >>> >>> net = SparseApplyFtrlNet() - >>> grad = Tensor(np.random.random(3, 3).astype(np.float32)) - >>> indices = Tnsor(np.ones([3]), mindspore.float32) + >>> grad = Tensor(np.random.rand(3, 3).astype(np.float32)) + >>> indices = Tensor(np.ones([3]), mindspore.int32) >>> output = net(grad, indices) """