From af74675b5a3129324229860e5370581a9bbb2d85 Mon Sep 17 00:00:00 2001 From: qingqing01 Date: Thu, 30 Jul 2020 10:53:29 +0800 Subject: [PATCH] Remove paddle.incubate.hapi.loss and reuse paddle.nn.layer.loss in high level API (#25590) * Remove paddle.incubate.hapi.loss and reuse the paddle.nn.layer.loss in high level API --- python/paddle/incubate/hapi/__init__.py | 2 - python/paddle/incubate/hapi/callbacks.py | 12 +- python/paddle/incubate/hapi/loss.py | 140 ------------------ python/paddle/incubate/hapi/metrics.py | 5 +- python/paddle/incubate/hapi/model.py | 119 +++++++++------ .../hapi/tests/dist_hapi_mnist_dynamic.py | 6 +- .../hapi/tests/dist_hapi_mnist_static.py | 6 +- .../paddle/incubate/hapi/tests/test_loss.py | 111 -------------- .../paddle/incubate/hapi/tests/test_model.py | 45 +++--- 9 files changed, 111 insertions(+), 335 deletions(-) delete mode 100644 python/paddle/incubate/hapi/loss.py delete mode 100644 python/paddle/incubate/hapi/tests/test_loss.py diff --git a/python/paddle/incubate/hapi/__init__.py b/python/paddle/incubate/hapi/__init__.py index 235460ae76a..94b6d1c6333 100644 --- a/python/paddle/incubate/hapi/__init__.py +++ b/python/paddle/incubate/hapi/__init__.py @@ -21,7 +21,6 @@ from . import model from .model import * from . import metrics -from . import loss from . import datasets from . import distributed from . import vision @@ -40,7 +39,6 @@ __all__ = [ 'distributed', 'download', 'metrics', - 'loss', 'vision', 'text', ] + model.__all__ + device.__all__ diff --git a/python/paddle/incubate/hapi/callbacks.py b/python/paddle/incubate/hapi/callbacks.py index 747108e2572..741552511f9 100644 --- a/python/paddle/incubate/hapi/callbacks.py +++ b/python/paddle/incubate/hapi/callbacks.py @@ -291,6 +291,7 @@ class ProgBarLogger(Callback): Examples: .. code-block:: python + import paddle import paddle.fluid as fluid import paddle.incubate.hapi as hapi @@ -299,11 +300,12 @@ class ProgBarLogger(Callback): train_dataset = hapi.datasets.MNIST(mode='train') - model = hapi.Model(hapi.vision.LeNet(), inputs, labels) + model = hapi.Model(hapi.vision.LeNet(classifier_activation=None), + inputs, labels) optim = fluid.optimizer.Adam(0.001) model.prepare(optimizer=optim, - loss_function=hapi.loss.CrossEntropy(), + loss_function=paddle.nn.CrossEntropyLoss(), metrics=hapi.metrics.Accuracy()) callback = hapi.callbacks.ProgBarLogger(log_freq=10) @@ -425,6 +427,7 @@ class ModelCheckpoint(Callback): Examples: .. code-block:: python + import paddle import paddle.fluid as fluid import paddle.incubate.hapi as hapi @@ -433,11 +436,12 @@ class ModelCheckpoint(Callback): train_dataset = hapi.datasets.MNIST(mode='train') - model = hapi.Model(hapi.vision.LeNet(), inputs, labels) + model = hapi.Model(hapi.vision.LeNet(classifier_activation=None), + inputs, labels) optim = fluid.optimizer.Adam(0.001) model.prepare(optimizer=optim, - loss_function=hapi.loss.CrossEntropy(), + loss_function=paddle.nn.CrossEntropyLoss(), metrics=hapi.metrics.Accuracy()) callback = hapi.callbacks.ModelCheckpoint(save_dir='./temp') diff --git a/python/paddle/incubate/hapi/loss.py b/python/paddle/incubate/hapi/loss.py deleted file mode 100644 index 72c8488d596..00000000000 --- a/python/paddle/incubate/hapi/loss.py +++ /dev/null @@ -1,140 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os - -from paddle import fluid -from paddle.fluid.framework import in_dygraph_mode, Variable -from paddle.fluid.dygraph.base import to_variable - -from .utils import to_list - -__all__ = ['Loss', 'CrossEntropy', 'SoftmaxWithCrossEntropy'] - - -class Loss(object): - """ - Base class for loss, encapsulates loss logic and APIs - - Usage: - custom_loss = CustomLoss() - loss = custom_loss(inputs, labels) - - Examples: - .. code-block:: python - - from paddle.incubate.hapi.loss import Loss - from paddle import fluid - - class SoftmaxWithCrossEntropy(Loss): - def __init__(self, average=True): - super(SoftmaxWithCrossEntropy, self).__init__(average) - - def forward(self, outputs, labels): - return [ - fluid.layers.softmax_with_cross_entropy( - o, l, return_softmax=False) for o, l in zip(outputs, labels) - ] - - """ - - def __init__(self, average=True): - super(Loss, self).__init__() - self.average = average - - def forward(self, outputs, labels): - raise NotImplementedError() - - def __call__(self, outputs, labels=None): - labels = to_list(labels) - if in_dygraph_mode() and labels: - labels = [to_variable(l) for l in labels] - losses = to_list(self.forward(to_list(outputs), labels)) - if self.average: - losses = [fluid.layers.reduce_mean(l) for l in losses] - else: - losses = [fluid.layers.reduce_sum(l) for l in losses] - return losses - - -class CrossEntropy(Loss): - """ - Args: - input (list[Variable]): Input tensor, the data type is float32, - float64, int32, int64. - label (list[Variable]): Label tensor, the data type is float32, - float64, int32, int64. - average (bool, optional): Indicate whether to average the loss, Default: True. - Returns: - list[Variable]: The tensor variable storing the cross_entropy_loss of inputs and labels. - - Examples: - .. code-block:: python - - import paddle.fluid as fluid - import paddle.incubate.hapi as hapi - - fluid.enable_dygraph() - - model = hapi.Model(hapi.vision.LeNet()) - model.prepare(loss_function=hapi.loss.CrossEntropy()) - - """ - - def __init__(self, average=True): - super(CrossEntropy, self).__init__(average) - - def forward(self, outputs, labels): - return [ - fluid.layers.cross_entropy(o, l) for o, l in zip(outputs, labels) - ] - - -class SoftmaxWithCrossEntropy(Loss): - """ - this op combined softmax and cross entropy. - Args: - input (list[Variable]): Input tensor, the data type is float32, - float64, int32, int64. - label (list[Variable]): Label tensor, the data type is float32, - float64, int32, int64. - average (bool, optional): Indicate whether to average the loss, Default: True. - Returns: - list[Variable]: The tensor variable storing the cross_entropy_loss of inputs and labels. - - Examples: - .. code-block:: python - - import paddle.fluid as fluid - import paddle.incubate.hapi as hapi - - fluid.enable_dygraph() - - model = hapi.Model(hapi.vision.LeNet(classifier_activation=None)) - loss = hapi.loss.SoftmaxWithCrossEntropy() - model.prepare(loss_function=loss) - """ - - def __init__(self, average=True): - super(SoftmaxWithCrossEntropy, self).__init__(average) - - def forward(self, outputs, labels): - return [ - fluid.layers.softmax_with_cross_entropy( - o, l, return_softmax=False) for o, l in zip(outputs, labels) - ] diff --git a/python/paddle/incubate/hapi/metrics.py b/python/paddle/incubate/hapi/metrics.py index 3b630f50b24..9e9a2e78524 100644 --- a/python/paddle/incubate/hapi/metrics.py +++ b/python/paddle/incubate/hapi/metrics.py @@ -170,6 +170,7 @@ class Accuracy(Metric): .. code-block:: python + import paddle import paddle.fluid as fluid import paddle.incubate.hapi as hapi @@ -177,12 +178,12 @@ class Accuracy(Metric): train_dataset = hapi.datasets.MNIST(mode='train') - model = hapi.Model(hapi.vision.LeNet()) + model = hapi.Model(hapi.vision.LeNet(classifier_activation=None)) optim = fluid.optimizer.Adam( learning_rate=0.001, parameter_list=model.parameters()) model.prepare( optim, - loss_function=hapi.loss.CrossEntropy(average=False), + loss_function=paddle.nn.CrossEntropyLoss(), metrics=hapi.metrics.Accuracy()) model.fit(train_dataset, batch_size=64) diff --git a/python/paddle/incubate/hapi/model.py b/python/paddle/incubate/hapi/model.py index c94cddd826f..0b12987b10a 100644 --- a/python/paddle/incubate/hapi/model.py +++ b/python/paddle/incubate/hapi/model.py @@ -35,7 +35,6 @@ from paddle.fluid.incubate.fleet.collective import fleet, DistributedStrategy from paddle.fluid.incubate.fleet.base import role_maker from paddle.io import DataLoader, Dataset -from .loss import Loss from .distributed import DistributedBatchSampler, _all_gather, prepare_distributed_context, _parallel_context_initialized from .metrics import Metric from .callbacks import config_callbacks @@ -327,7 +326,7 @@ class StaticGraphAdapter(object): rets = [np.array(v) for v in rets] if self.mode == 'test': return rets[:] - losses = rets[:num_loss] + metric_states = restore_flatten_list(rets[num_loss:], metric_splits) metrics = [] for metric, state in zip(self.model._metrics, metric_states): @@ -351,7 +350,11 @@ class StaticGraphAdapter(object): self._merge_count[self.mode + '_batch'] = samples metrics.append(metric.update(*state)) - return (losses, metrics) if len(metrics) > 0 else losses + + if num_loss and len(metrics): + return rets[:num_loss], metrics + else: + return rets[:num_loss] if num_loss else metrics def prepare(self): modes = ['train', 'eval', 'test'] @@ -383,15 +386,15 @@ class StaticGraphAdapter(object): losses = [] metrics = [] with fluid.program_guard(prog, self._startup_prog): - ins = self.model._inputs - lbls = self.model._labels if self.model._labels else [] - inputs = [k.forward() for k in to_list(ins)] - labels = [k.forward() for k in to_list(lbls)] + inputs = self.model._inputs + labels = self.model._labels if self.model._labels else [] + inputs = [k.forward() for k in to_list(inputs)] + labels = [k.forward() for k in to_list(labels)] self._label_vars[mode] = labels outputs = to_list(self.model.network.forward(*inputs)) if mode != 'test' and self.model._loss_function: - losses = self.model._loss_function(outputs, labels) + losses = self.model._loss_function(*(outputs + labels)) if self._nranks > 1 and mode != 'train': outputs = [_all_gather(o, self._nranks) for o in outputs] @@ -424,7 +427,7 @@ class StaticGraphAdapter(object): self._progs[mode] = prog self._endpoints[mode] = { "output": outputs, - "loss": losses, + "loss": to_list(losses), "metric": metrics } @@ -501,11 +504,13 @@ class DynamicGraphAdapter(object): self.model.network.train() self.mode = 'train' inputs = to_list(inputs) - if labels is not None: - labels = [to_variable(l) for l in to_list(labels)] + labels = labels or [] + labels = [to_variable(l) for l in to_list(labels)] + if self._nranks > 1: outputs = self.ddp_model.forward(* [to_variable(x) for x in inputs]) - losses = self.model._loss_function(outputs, labels) + losses = self.model._loss_function(*(to_list(outputs) + labels)) + losses = to_list(losses) final_loss = fluid.layers.sum(losses) final_loss = self.ddp_model.scale_loss(final_loss) final_loss.backward() @@ -513,7 +518,8 @@ class DynamicGraphAdapter(object): else: outputs = self.model.network.forward( * [to_variable(x) for x in inputs]) - losses = self.model._loss_function(outputs, labels) + losses = self.model._loss_function(*(to_list(outputs) + labels)) + losses = to_list(losses) final_loss = fluid.layers.sum(losses) final_loss.backward() @@ -521,8 +527,7 @@ class DynamicGraphAdapter(object): self.model.network.clear_gradients() metrics = [] for metric in self.model._metrics: - metric_outs = metric.add_metric_op(*(to_list(outputs) + to_list( - labels))) + metric_outs = metric.add_metric_op(*(to_list(outputs) + labels)) m = metric.update(* [to_numpy(m) for m in to_list(metric_outs)]) metrics.append(m) @@ -533,13 +538,14 @@ class DynamicGraphAdapter(object): self.model.network.eval() self.mode = 'eval' inputs = to_list(inputs) - if labels is not None: - labels = [to_variable(l) for l in to_list(labels)] + labels = labels or [] + labels = [to_variable(l) for l in to_list(labels)] + outputs = self.model.network.forward(* [to_variable(x) for x in inputs]) if self.model._loss_function: - losses = self.model._loss_function(outputs, labels) - else: - losses = [] + losses = self.model._loss_function(*(to_list(outputs) + labels)) + losses = to_list(losses) + if self._nranks > 1: outputs = [_all_gather(o, self._nranks) for o in to_list(outputs)] labels = [_all_gather(l, self._nranks) for l in labels] @@ -565,15 +571,16 @@ class DynamicGraphAdapter(object): self._merge_count[self.mode + '_total'] += samples self._merge_count[self.mode + '_batch'] = samples - metric_outs = metric.add_metric_op(*(to_list(outputs) + to_list( - labels))) + metric_outs = metric.add_metric_op(*(to_list(outputs) + labels)) m = metric.update(* [to_numpy(m) for m in to_list(metric_outs)]) metrics.append(m) - # To be consistent with static graph - # return empty loss if loss_function is None - return ([to_numpy(l) for l in losses], metrics) \ - if len(metrics) > 0 else [to_numpy(l) for l in losses] + if self.model._loss_function and len(metrics): + return [to_numpy(l) for l in losses], metrics + elif self.model._loss_function: + return [to_numpy(l) for l in losses] + else: + return metrics def test_batch(self, inputs): self.model.network.eval() @@ -679,13 +686,15 @@ class Model(object): Usage: .. code-block:: python + import paddle import paddle.fluid as fluid import paddle.incubate.hapi as hapi class MyNet(fluid.dygraph.Layer): - def __init__(self): + def __init__(self, classifier_act=None): super(MyNet, self).__init__() - self._fc1 = fluid.dygraph.Linear(784, 200, act='softmax') + self._fc1 = fluid.dygraph.Linear(784, 200, act=classifier_act) + def forward(self, x): y = self._fc1(x) return y @@ -702,7 +711,7 @@ class Model(object): optim = fluid.optimizer.SGD(learning_rate=1e-3, parameter_list=model.parameters()) model.prepare(optim, - hapi.loss.CrossEntropy(average=True), + paddle.nn.CrossEntropyLoss(), hapi.metrics.Accuracy()) mnist_data = hapi.datasets.MNIST(mode='train', chw_format=False) @@ -762,13 +771,15 @@ class Model(object): .. code-block:: python import numpy as np + import paddle import paddle.fluid as fluid import paddle.incubate.hapi as hapi class MyNet(fluid.dygraph.Layer): - def __init__(self): + def __init__(self, classifier_act=None): super(MyNet, self).__init__() - self._fc = fluid.dygraph.Linear(784, 10, act='softmax') + self._fc = fluid.dygraph.Linear(784, 10, act=classifier_act) + def forward(self, x): y = self._fc(x) return y @@ -781,7 +792,7 @@ class Model(object): model = hapi.Model(MyNet(), input, label) optim = fluid.optimizer.SGD(learning_rate=1e-3, parameter_list=model.parameters()) - model.prepare(optim, hapi.loss.CrossEntropy(average=True)) + model.prepare(optim, paddle.nn.CrossEntropyLoss()) data = np.random.random(size=(4,784)).astype(np.float32) label = np.random.randint(0, 10, size=(4, 1)).astype(np.int64) loss = model.train_batch([data], [label]) @@ -809,13 +820,15 @@ class Model(object): .. code-block:: python import numpy as np + import paddle import paddle.fluid as fluid import paddle.incubate.hapi as hapi class MyNet(fluid.dygraph.Layer): - def __init__(self): + def __init__(self, classifier_act=None): super(MyNet, self).__init__() - self._fc = fluid.dygraph.Linear(784, 10, act='softmax') + self._fc = fluid.dygraph.Linear(784, 10, act=classifier_act) + def forward(self, x): y = self._fc(x) return y @@ -829,7 +842,7 @@ class Model(object): optim = fluid.optimizer.SGD(learning_rate=1e-3, parameter_list=model.parameters()) model.prepare(optim, - hapi.loss.CrossEntropy(average=True)) + paddle.nn.CrossEntropyLoss()) data = np.random.random(size=(4,784)).astype(np.float32) label = np.random.randint(0, 10, size=(4, 1)).astype(np.int64) loss = model.eval_batch([data], [label]) @@ -1054,9 +1067,10 @@ class Model(object): optimizer (Optimizer|None): Optimizer must be set in training and should be a Optimizer instance. It can be None in eval and test mode. - loss_function (Loss|None): Loss function must be set in training - and should be a Loss instance. It can be None when there is - no loss. + loss_function (Loss|callable function|None): Loss function can + be a `fluid.dygraph.Layer` instance or any callable function + taken the predicted values and ground truth values as input. + It can be None when there is no loss. metrics (Metric|list of Metric|None): If metrics is set, all metrics will be calculated and output in train/eval mode. @@ -1086,8 +1100,10 @@ class Model(object): self._optimizer = optimizer if loss_function: - if not isinstance(loss_function, Loss): - raise TypeError("'loss_function' must be sub classes of 'Loss'") + if not isinstance(loss_function, fluid.dygraph.Layer) or \ + not callable(loss_function): + raise TypeError("'loss_function' must be sub classes of \ + `fluid.dygraph.Layer` or any callable function.") self._loss_function = loss_function metrics = metrics or [] @@ -1167,6 +1183,7 @@ class Model(object): .. code-block:: python + import paddle import paddle.fluid as fluid import paddle.incubate.hapi as hapi @@ -1180,12 +1197,13 @@ class Model(object): input = hapi.Input('image', [None, 1, 28, 28], 'float32') label = hapi.Input('label', [None, 1], 'int64') - model = hapi.Model(hapi.vision.LeNet(), input, label) + model = hapi.Model(hapi.vision.LeNet(classifier_activation=None), + input, label) optim = fluid.optimizer.Adam( learning_rate=0.001, parameter_list=model.parameters()) model.prepare( optim, - hapi.loss.CrossEntropy(), + paddle.nn.CrossEntropyLoss(), hapi.metrics.Accuracy(topk=(1, 2))) model.fit(train_dataset, val_dataset, @@ -1198,6 +1216,7 @@ class Model(object): .. code-block:: python + import paddle import paddle.fluid as fluid import paddle.incubate.hapi as hapi @@ -1215,12 +1234,13 @@ class Model(object): input = hapi.Input('image', [None, 1, 28, 28], 'float32') label = hapi.Input('label', [None, 1], 'int64') - model = hapi.Model(hapi.vision.LeNet(), input, label) + model = hapi.Model(hapi.vision.LeNet(classifier_activation=None), + input, label) optim = fluid.optimizer.Adam( learning_rate=0.001, parameter_list=model.parameters()) model.prepare( optim, - hapi.loss.CrossEntropy(), + paddle.nn.CrossEntropyLoss(), hapi.metrics.Accuracy(topk=(1, 2))) model.fit(train_loader, val_loader, @@ -1581,9 +1601,12 @@ class Model(object): if mode != 'test': outs = getattr(self, mode + '_batch')(data[:len(self._inputs)], data[len(self._inputs):]) - # losses - loss = outs[0] if self._metrics else outs - metrics = [[l[0] for l in loss]] + if self._metrics and self._loss_function: + metrics = [[l[0] for l in outs[0]]] + elif self._loss_function: + metrics = [[l[0] for l in outs]] + else: + metrics = [] # metrics for metric in self._metrics: @@ -1621,7 +1644,7 @@ class Model(object): metric.reset() def _metrics_name(self): - metrics_name = ['loss'] + metrics_name = ['loss'] if self._loss_function else [] for m in self._metrics: metrics_name.extend(to_list(m.name())) return metrics_name diff --git a/python/paddle/incubate/hapi/tests/dist_hapi_mnist_dynamic.py b/python/paddle/incubate/hapi/tests/dist_hapi_mnist_dynamic.py index cbb41d0bbb9..b338f3310b4 100644 --- a/python/paddle/incubate/hapi/tests/dist_hapi_mnist_dynamic.py +++ b/python/paddle/incubate/hapi/tests/dist_hapi_mnist_dynamic.py @@ -23,7 +23,7 @@ import contextlib from paddle import fluid from paddle.incubate.hapi import Model, Input, set_device -from paddle.incubate.hapi.loss import CrossEntropy +from paddle.nn.layer.loss import CrossEntropyLoss from paddle.incubate.hapi.vision.models import LeNet from paddle.incubate.hapi.metrics import Accuracy from paddle.incubate.hapi.callbacks import ProgBarLogger @@ -67,10 +67,10 @@ class TestDistTraning(unittest.TestCase): inputs = [Input('image', im_shape, 'float32')] labels = [Input('label', [None, 1], 'int64')] - model = Model(LeNet(), inputs, labels) + model = Model(LeNet(classifier_activation=None), inputs, labels) optim = fluid.optimizer.Momentum( learning_rate=0.001, momentum=.9, parameter_list=model.parameters()) - model.prepare(optim, CrossEntropy(), Accuracy()) + model.prepare(optim, CrossEntropyLoss(), Accuracy()) train_dataset = MnistDataset(mode='train') val_dataset = MnistDataset(mode='test') diff --git a/python/paddle/incubate/hapi/tests/dist_hapi_mnist_static.py b/python/paddle/incubate/hapi/tests/dist_hapi_mnist_static.py index e407dd12d56..1484620a4ef 100644 --- a/python/paddle/incubate/hapi/tests/dist_hapi_mnist_static.py +++ b/python/paddle/incubate/hapi/tests/dist_hapi_mnist_static.py @@ -23,7 +23,7 @@ import contextlib from paddle import fluid from paddle.incubate.hapi import Model, Input, set_device -from paddle.incubate.hapi.loss import CrossEntropy +from paddle.nn.layer.loss import CrossEntropyLoss from paddle.incubate.hapi.vision.models import LeNet from paddle.incubate.hapi.metrics import Accuracy from paddle.incubate.hapi.callbacks import ProgBarLogger @@ -66,10 +66,10 @@ class TestDistTraning(unittest.TestCase): inputs = [Input('image', im_shape, 'float32')] labels = [Input('label', [None, 1], 'int64')] - model = Model(LeNet(), inputs, labels) + model = Model(LeNet(classifier_activation=None), inputs, labels) optim = fluid.optimizer.Momentum( learning_rate=0.001, momentum=.9, parameter_list=model.parameters()) - model.prepare(optim, CrossEntropy(), Accuracy()) + model.prepare(optim, CrossEntropyLoss(), Accuracy()) train_dataset = MnistDataset(mode='train') val_dataset = MnistDataset(mode='test') diff --git a/python/paddle/incubate/hapi/tests/test_loss.py b/python/paddle/incubate/hapi/tests/test_loss.py deleted file mode 100644 index f729b38b81f..00000000000 --- a/python/paddle/incubate/hapi/tests/test_loss.py +++ /dev/null @@ -1,111 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import division -from __future__ import print_function - -import unittest -import os -import six -import numpy as np -import shutil -import copy - -import paddle -from paddle import fluid - -from paddle.incubate.hapi.model import Model, Input -from paddle.incubate.hapi.loss import CrossEntropy, SoftmaxWithCrossEntropy - - -def stable_softmax(x): - """Compute the softmax of vector x in a numerically stable way.""" - # clip to shiftx, otherwise, when calc loss with - # log(exp(shiftx)), may get log(0)=INF - shiftx = (x - np.max(x)).clip(-64.) - exps = np.exp(shiftx) - return exps / np.sum(exps) - - -def randomize_probability(batch_size, class_num, dtype='float32'): - prob = np.random.uniform( - 0.1, 1.0, size=(batch_size, class_num)).astype(dtype) - prob_sum = prob.sum(axis=1) - for i in six.moves.xrange(len(prob)): - prob[i] /= prob_sum[i] - return prob - - -def numpy_ce(x, label): - return np.asmatrix( - [[-np.log(x[i][label[i][0]])] for i in range(x.shape[0])], - dtype="float32").mean() - - -class TestLoss(unittest.TestCase): - def test_cross_entropy(self): - class_num = 100 - batch_size = 128 - inputs = [randomize_probability(128, class_num) for _ in range(2)] - - labels = [ - np.random.randint( - 0, class_num, (batch_size, 1), dtype="int64") for _ in range(2) - ] - - gt_out = [numpy_ce(inputs[i], labels[i]) for i in range(2)] - - fluid.enable_dygraph() - cross_entropy = CrossEntropy() - out = cross_entropy( - [fluid.dygraph.to_variable(x) for x in inputs], - [fluid.dygraph.to_variable(label) for label in labels]) - out = [o.numpy() for o in out] - - for o, g in zip(out, gt_out): - np.testing.assert_allclose(o, g, atol=1e-5) - - def test_soft_cross_entronpy(self): - class_num = 100 - batch_size = 128 - - inputs = [randomize_probability(128, class_num) for _ in range(2)] - - labels = [ - np.random.randint( - 0, class_num, (batch_size, 1), dtype="int64") for _ in range(2) - ] - - fluid.enable_dygraph() - softmax_cross_entropy = SoftmaxWithCrossEntropy() - - softmax_cross_entropy( - [fluid.dygraph.to_variable(x) for x in inputs], - [fluid.dygraph.to_variable(label) for label in labels]) - - softmax_cross_entropy = SoftmaxWithCrossEntropy(average=False) - - inputs = [randomize_probability(128, class_num)] - - labels = [ - np.random.randint( - 0, class_num, (batch_size, 1), dtype="int64") - ] - - softmax_cross_entropy([fluid.dygraph.to_variable(x) for x in inputs], - fluid.dygraph.to_variable(labels[0])) - - -if __name__ == '__main__': - unittest.main() diff --git a/python/paddle/incubate/hapi/tests/test_model.py b/python/paddle/incubate/hapi/tests/test_model.py index 6cbdf7498db..f8be2e24256 100644 --- a/python/paddle/incubate/hapi/tests/test_model.py +++ b/python/paddle/incubate/hapi/tests/test_model.py @@ -28,7 +28,7 @@ from paddle.fluid.dygraph.base import to_variable import paddle.incubate.hapi as hapi from paddle.incubate.hapi import Model, Input -from paddle.incubate.hapi.loss import CrossEntropy +from paddle.nn.layer.loss import CrossEntropyLoss from paddle.incubate.hapi.metrics import Accuracy from paddle.incubate.hapi.datasets import MNIST from paddle.incubate.hapi.vision.models import LeNet @@ -36,7 +36,7 @@ from paddle.incubate.hapi.distributed import DistributedBatchSampler, prepare_di class LeNetDygraph(fluid.dygraph.Layer): - def __init__(self, num_classes=10, classifier_activation='softmax'): + def __init__(self, num_classes=10, classifier_activation=None): super(LeNetDygraph, self).__init__() self.num_classes = num_classes self.features = Sequential( @@ -97,7 +97,7 @@ def dynamic_train(model, dataloader): model.train() for inputs, labels in dataloader: outputs = model(inputs) - loss = fluid.layers.cross_entropy(outputs, labels) + loss = CrossEntropyLoss(reduction="sum")(outputs, labels) avg_loss = fluid.layers.reduce_sum(loss) avg_loss.backward() optim.minimize(avg_loss) @@ -190,13 +190,13 @@ class TestModel(unittest.TestCase): fluid.default_startup_program().random_seed = seed fluid.default_main_program().random_seed = seed - net = LeNet() + net = LeNet(classifier_activation=None) optim_new = fluid.optimizer.Adam( learning_rate=0.001, parameter_list=net.parameters()) model = Model(net, inputs=self.inputs, labels=self.labels) model.prepare( optim_new, - loss_function=CrossEntropy(average=False), + loss_function=CrossEntropyLoss(reduction="sum"), metrics=Accuracy()) model.fit(self.train_dataset, batch_size=64, shuffle=False) @@ -271,9 +271,9 @@ class TestModel(unittest.TestCase): class MyModel(fluid.dygraph.Layer): - def __init__(self): + def __init__(self, classifier_activation='softmax'): super(MyModel, self).__init__() - self._fc = Linear(20, 10, act='softmax') + self._fc = Linear(20, 10, act=classifier_activation) def forward(self, x): y = self._fc(x) @@ -293,13 +293,12 @@ class TestModelFunction(unittest.TestCase): def get_expect(): fluid.enable_dygraph(fluid.CPUPlace()) self.set_seed() - m = MyModel() + m = MyModel(classifier_activation=None) optim = fluid.optimizer.SGD(learning_rate=0.001, parameter_list=m.parameters()) m.train() output = m(to_variable(data)) - l = to_variable(label) - loss = fluid.layers.cross_entropy(output, l) + loss = CrossEntropyLoss(reduction='sum')(output, to_variable(label)) avg_loss = fluid.layers.reduce_sum(loss) avg_loss.backward() optim.minimize(avg_loss) @@ -313,14 +312,15 @@ class TestModelFunction(unittest.TestCase): fluid.enable_dygraph(device) if dynamic else None self.set_seed() - net = MyModel() + net = MyModel(classifier_activation=None) optim2 = fluid.optimizer.SGD(learning_rate=0.001, parameter_list=net.parameters()) inputs = [Input('x', [None, dim], 'float32')] labels = [Input('label', [None, 1], 'int64')] model = Model(net, inputs, labels) - model.prepare(optim2, loss_function=CrossEntropy(average=False)) + model.prepare( + optim2, loss_function=CrossEntropyLoss(reduction="sum")) loss, = model.train_batch([data], [label]) np.testing.assert_allclose(loss.flatten(), ref.flatten()) @@ -358,14 +358,15 @@ class TestModelFunction(unittest.TestCase): for dynamic in [True, False]: device = hapi.set_device('cpu') fluid.enable_dygraph(device) if dynamic else None - net = MyModel() + net = MyModel(classifier_activation=None) inputs = [Input('x', [None, 20], 'float32')] labels = [Input('label', [None, 1], 'int64')] optim = fluid.optimizer.SGD(learning_rate=0.001, parameter_list=net.parameters()) model = Model(net, inputs, labels) model.prepare( - optimizer=optim, loss_function=CrossEntropy(average=False)) + optimizer=optim, + loss_function=CrossEntropyLoss(reduction="sum")) model.save(path + '/test') model.load(path + '/test') shutil.rmtree(path) @@ -376,48 +377,48 @@ class TestModelFunction(unittest.TestCase): # dynamic saving device = hapi.set_device('cpu') fluid.enable_dygraph(device) - model = Model(MyModel()) + model = Model(MyModel(classifier_activation=None)) optim = fluid.optimizer.SGD(learning_rate=0.001, parameter_list=model.parameters()) model.prepare( - optimizer=optim, loss_function=CrossEntropy(average=False)) + optimizer=optim, loss_function=CrossEntropyLoss(reduction="sum")) model.save(path + '/test') fluid.disable_dygraph() inputs = [Input('x', [None, 20], 'float32')] labels = [Input('label', [None, 1], 'int64')] - model = Model(MyModel(), inputs, labels) + model = Model(MyModel(classifier_activation=None), inputs, labels) optim = fluid.optimizer.SGD(learning_rate=0.001, parameter_list=model.parameters()) model.prepare( - optimizer=optim, loss_function=CrossEntropy(average=False)) + optimizer=optim, loss_function=CrossEntropyLoss(reduction="sum")) model.load(path + '/test') shutil.rmtree(path) def test_static_save_dynamic_load(self): path = tempfile.mkdtemp() - net = MyModel() + net = MyModel(classifier_activation=None) inputs = [Input('x', [None, 20], 'float32')] labels = [Input('label', [None, 1], 'int64')] optim = fluid.optimizer.SGD(learning_rate=0.001, parameter_list=net.parameters()) model = Model(net, inputs, labels) model.prepare( - optimizer=optim, loss_function=CrossEntropy(average=False)) + optimizer=optim, loss_function=CrossEntropyLoss(reduction="sum")) model.save(path + '/test') device = hapi.set_device('cpu') fluid.enable_dygraph(device) #if dynamic else None - net = MyModel() + net = MyModel(classifier_activation=None) inputs = [Input('x', [None, 20], 'float32')] labels = [Input('label', [None, 1], 'int64')] optim = fluid.optimizer.SGD(learning_rate=0.001, parameter_list=net.parameters()) model = Model(net, inputs, labels) model.prepare( - optimizer=optim, loss_function=CrossEntropy(average=False)) + optimizer=optim, loss_function=CrossEntropyLoss(reduction="sum")) model.load(path + '/test') shutil.rmtree(path) fluid.disable_dygraph() -- GitLab