diff --git a/metrics/metric.py b/metrics.py similarity index 66% rename from metrics/metric.py rename to metrics.py index 185328efc093ea8f3ad8682ca55ff577041ecd0f..b9cb49ce09b84dda6c0535232f191e4a3af437a8 100644 --- a/metrics/metric.py +++ b/metrics.py @@ -16,13 +16,14 @@ from __future__ import absolute_import import six import abc +import numpy as np import logging FORMAT = '%(asctime)s-%(levelname)s: %(message)s' logging.basicConfig(level=logging.INFO, format=FORMAT) logger = logging.getLogger(__name__) -__all__ = ['Metric'] +__all__ = ['Metric', 'Accuracy'] @six.add_metaclass(abc.ABCMeta) @@ -58,3 +59,31 @@ class Metric(object): """ raise NotImplementedError("function 'accumulate' not implemented in {}.".format(self.__class__.__name__)) + +class Accuracy(Metric): + """ + Encapsulates accuracy metric logic + """ + + def __init__(self, topk=(1, ), *args, **kwargs): + super(Accuracy, self).__init__(*args, **kwargs) + self.topk = topk + self.maxk = max(topk) + self.reset() + + def update(self, pred, label, *args, **kwargs): + pred = np.argsort(pred[0])[:, ::-1][:, :self.maxk] + corr = (pred == np.repeat(label[0], self.maxk, 1)) + self.correct = np.append(self.correct, corr, axis=0) + + def reset(self): + self.correct = np.empty((0, self.maxk), dtype="int32") + + def accumulate(self): + res = [] + num_samples = self.correct.shape[0] + for k in self.topk: + correct_k = self.correct[:, :k].sum() + res.append(round(100.0 * correct_k / num_samples, 2)) + return res + diff --git a/mnist.py b/mnist.py index 2f6d95c3fc54c40f859c9ad49fc7477b2ffa0b26..91fa18d7064fe193e13e4f2e2c3d1c73bd2e67b0 100644 --- a/mnist.py +++ b/mnist.py @@ -27,6 +27,7 @@ from paddle.fluid.optimizer import Momentum from paddle.fluid.dygraph.nn import Conv2D, Pool2D, Linear from model import Model, CrossEntropy +from metrics import Accuracy class SimpleImgConvPool(fluid.dygraph.Layer): @@ -143,7 +144,7 @@ def main(): model = MNIST() optim = Momentum(learning_rate=FLAGS.lr, momentum=.9, parameter_list=model.parameters()) - model.prepare(optim, CrossEntropy()) + model.prepare(optim, CrossEntropy(), metrics=Accuracy(topk=(1, 2))) if FLAGS.resume is not None: model.load(FLAGS.resume) @@ -163,6 +164,10 @@ def main(): if idx % 10 == 0: print("{:04d}: loss {:0.3f} top1: {:0.3f}%".format( idx, train_loss / (idx + 1), train_acc / (idx + 1))) + for metric in model._metrics: + res = metric.accumulate() + print("train epoch {:03d}: top1: {:0.3f}%, top2: {:0.3f}".format(e, res[0], res[1])) + metric.reset() print("======== eval epoch {} ========".format(e)) for idx, batch in enumerate(val_loader()): @@ -175,6 +180,10 @@ def main(): if idx % 10 == 0: print("{:04d}: loss {:0.3f} top1: {:0.3f}%".format( idx, val_loss / (idx + 1), val_acc / (idx + 1))) + for metric in model._metrics: + res = metric.accumulate() + print("eval epoch {:03d}: top1: {:0.3f}%, top2: {:0.3f}".format(e, res[0], res[1])) + metric.reset() model.save('mnist_checkpoints/{:02d}'.format(e)) diff --git a/model.py b/model.py index 74165badc644c38e0ae7109bccd1c2d9f9258656..3949e318369bc53e9782d79dd47e66074ac65c3a 100644 --- a/model.py +++ b/model.py @@ -26,7 +26,7 @@ from paddle.fluid.framework import in_dygraph_mode, Variable from paddle.fluid.executor import global_scope from paddle.fluid.io import is_belong_to_optimizer from paddle.fluid.dygraph.base import to_variable -from metrics.metric import Metric +from metrics import Metric __all__ = ['shape_hints', 'Model', 'Loss', 'CrossEntropy'] @@ -297,6 +297,8 @@ class StaticGraphAdapter(object): metric.update(outputs, labels) return outputs, losses else: # train + for metric in self.model._metrics: + metric.update(outputs, labels) return outputs, losses def _make_program(self, inputs): @@ -311,7 +313,7 @@ class StaticGraphAdapter(object): if self.mode != 'test': label_vars = self._infer_label_vars(outputs) self._label_vars[self.mode] = label_vars - losses = self.model._loss_function(outputs[0], label_vars) + losses = self.model._loss_function(outputs, label_vars) if self.mode == 'train': self._loss_endpoint = fluid.layers.sum(losses) self.model._optimizer.minimize(self._loss_endpoint) @@ -319,7 +321,7 @@ class StaticGraphAdapter(object): prog = prog.clone(for_test=True) self._progs[self.mode] = prog self._endpoints[self.mode] = { - "output": outputs[1:], + "output": outputs, "label": label_vars, "loss": losses, } @@ -419,12 +421,14 @@ class DynamicGraphAdapter(object): self.mode = 'train' inputs = to_list(inputs) labels = to_list(labels) - outputs = self.model.forward(*[to_variable(x) for x in inputs])[0] + outputs = self.model.forward(*[to_variable(x) for x in inputs]) losses = self.model._loss_function(outputs, labels) final_loss = fluid.layers.sum(losses) final_loss.backward() self.model._optimizer.minimize(final_loss) self.model.clear_gradients() + for metric in self.model._metrics: + metric.update([to_numpy(o) for o in to_list(outputs)], labels) return [to_numpy(o) for o in to_list(outputs)], \ [to_numpy(l) for l in losses] @@ -436,18 +440,18 @@ class DynamicGraphAdapter(object): inputs = to_list(inputs) labels = to_list(labels) outputs = self.model.forward(*[to_variable(x) for x in inputs]) - losses = self.model._loss_function(outputs[0], labels) + losses = self.model._loss_function(outputs, labels) for metric in self.model._metrics: - metric.update([to_numpy(o) for o in outputs[1:]], labels) - return [to_numpy(o) for o in to_list(outputs[0])], \ + metric.update([to_numpy(o) for o in to_list(outputs)], labels) + return [to_numpy(o) for o in to_list(outputs)], \ [to_numpy(l) for l in losses] def test(self, inputs, device='CPU', device_ids=None): super(Model, self.model).eval() self.mode = 'test' inputs = [to_variable(x) for x in to_list(inputs)] - outputs = self.model.forward(*inputs)[1:] - return [to_numpy(o) for o in to_list(outputs[1:])] + outputs = self.model.forward(*inputs) + return [to_numpy(o) for o in to_list(outputs)] def parameters(self, *args, **kwargs): return super(Model, self.model).parameters(*args, **kwargs) diff --git a/yolov3.py b/yolov3.py index 5806675ee6e0e1a559cc6cb43dea6aae77a869f7..dbebf34243419cefef85b636c04e971428045bb5 100644 --- a/yolov3.py +++ b/yolov3.py @@ -35,7 +35,6 @@ from paddle.fluid.regularizer import L2Decay from model import Model, Loss, shape_hints from resnet import ResNet, ConvBNLayer -from metrics.coco import COCOMetric import logging FORMAT = '%(asctime)s-%(levelname)s: %(message)s' @@ -174,7 +173,7 @@ class YOLOv3(Model): route = self.route_blocks[idx](route) route = fluid.layers.resize_nearest(route, scale=2) - if self.mode != 'train': + if self.mode == 'test': anchor_mask = self.anchor_masks[idx] mask_anchors = [] for m in anchor_mask: @@ -195,10 +194,10 @@ class YOLOv3(Model): downsample //= 2 - if self.mode == 'train': - return [outputs] + if self.mode != 'test': + return outputs - return [outputs, img_id, fluid.layers.multiclass_nms( + return [img_id, fluid.layers.multiclass_nms( bboxes=fluid.layers.concat(boxes, axis=1), scores=fluid.layers.concat(scores, axis=2), score_threshold=self.valid_thresh, @@ -533,12 +532,15 @@ def main(): anno_path = os.path.join(FLAGS.data, 'annotations', 'instances_val2017.json') model.prepare(optim, YoloLoss(num_classes=NUM_CLASSES), - metrics=COCOMetric(anno_path, with_background=False)) + # For YOLOv3, output variable in train/eval is different, + # which is not supported by metric, add by callback later? + # metrics=COCOMetric(anno_path, with_background=False) + ) for e in range(epoch): - # logger.info("======== train epoch {} ========".format(e)) - # run(model, train_loader) - # model.save('yolo_checkpoints/{:02d}'.format(e)) + logger.info("======== train epoch {} ========".format(e)) + run(model, train_loader) + model.save('yolo_checkpoints/{:02d}'.format(e)) logger.info("======== eval epoch {} ========".format(e)) run(model, val_loader, mode='eval') # should be called in fit() diff --git a/metrics/__init__.py b/yolov3/__init__.py similarity index 93% rename from metrics/__init__.py rename to yolov3/__init__.py index 340a1ac4a99dc3a25aa1c3fa5c58224e6e902fa0..9118340d83fefa17d4a7e8fc577ee22a2d3a2656 100644 --- a/metrics/__init__.py +++ b/yolov3/__init__.py @@ -12,5 +12,3 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import - diff --git a/metrics/coco.py b/yolov3/coco_metric.py similarity index 97% rename from metrics/coco.py rename to yolov3/coco_metric.py index e76950ad082beb74a1d2353c2c26c557853aaba2..ec7bcac24b3dde91d3ae85e39e7bf9e5151f43ec 100644 --- a/metrics/coco.py +++ b/yolov3/coco_metric.py @@ -12,14 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import - import sys import json from pycocotools.cocoeval import COCOeval from pycocotools.coco import COCO -from .metric import Metric +from metrics import Metric import logging FORMAT = '%(asctime)s-%(levelname)s: %(message)s' @@ -32,6 +30,7 @@ __all__ = ['COCOMetric'] OUTFILE = './bbox.json' +# considered to change to a callback later class COCOMetric(Metric): """ Metrci for MS-COCO dataset, only support update with batch