mv COCOMetric to yolov3/coco_metric

metrics/metric.py → metrics.py

@@ -16,13 +16,14 @@ from __future__ import absolute_import
 
 import six
 import abc
+import numpy as np
 
 import logging
 FORMAT = '%(asctime)s-%(levelname)s: %(message)s'
 logging.basicConfig(level=logging.INFO, format=FORMAT)
 logger = logging.getLogger(__name__)
 
-__all__ = ['Metric']
+__all__ = ['Metric', 'Accuracy']
 
 
 @six.add_metaclass(abc.ABCMeta)
@@ -58,3 +59,31 @@ class Metric(object):
         """
         raise NotImplementedError("function 'accumulate' not implemented in {}.".format(self.__class__.__name__))
 
+
+class Accuracy(Metric):
+    """
+    Encapsulates accuracy metric logic
+    """
+
+    def __init__(self, topk=(1, ), *args, **kwargs):
+       super(Accuracy, self).__init__(*args, **kwargs) 
+       self.topk = topk
+       self.maxk = max(topk)
+       self.reset()
+
+    def update(self, pred, label, *args, **kwargs):
+        pred = np.argsort(pred[0])[:, ::-1][:, :self.maxk]
+        corr = (pred == np.repeat(label[0], self.maxk, 1))
+        self.correct = np.append(self.correct, corr, axis=0)
+
+    def reset(self):
+       self.correct = np.empty((0, self.maxk), dtype="int32")
+
+    def accumulate(self):
+        res = []
+        num_samples = self.correct.shape[0]
+        for k in self.topk:
+            correct_k = self.correct[:, :k].sum()
+            res.append(round(100.0 * correct_k / num_samples, 2))
+        return res
+

mnist.py

@@ -27,6 +27,7 @@ from paddle.fluid.optimizer import Momentum
 from paddle.fluid.dygraph.nn import Conv2D, Pool2D, Linear
 
 from model import Model, CrossEntropy
+from metrics import Accuracy
 
 
 class SimpleImgConvPool(fluid.dygraph.Layer):
@@ -143,7 +144,7 @@ def main():
         model = MNIST()
         optim = Momentum(learning_rate=FLAGS.lr, momentum=.9,
                          parameter_list=model.parameters())
-        model.prepare(optim, CrossEntropy())
+        model.prepare(optim, CrossEntropy(), metrics=Accuracy(topk=(1, 2)))
         if FLAGS.resume is not None:
             model.load(FLAGS.resume)
 
@@ -163,6 +164,10 @@ def main():
                 if idx % 10 == 0:
                     print("{:04d}: loss {:0.3f} top1: {:0.3f}%".format(
                         idx, train_loss / (idx + 1), train_acc / (idx + 1)))
+            for metric in model._metrics:
+                res = metric.accumulate()
+                print("train epoch {:03d}: top1: {:0.3f}%, top2: {:0.3f}".format(e, res[0], res[1]))
+                metric.reset()
 
             print("======== eval epoch {} ========".format(e))
             for idx, batch in enumerate(val_loader()):
@@ -175,6 +180,10 @@ def main():
                 if idx % 10 == 0:
                     print("{:04d}: loss {:0.3f} top1: {:0.3f}%".format(
                         idx, val_loss / (idx + 1), val_acc / (idx + 1)))
+            for metric in model._metrics:
+                res = metric.accumulate()
+                print("eval epoch {:03d}: top1: {:0.3f}%, top2: {:0.3f}".format(e, res[0], res[1]))
+                metric.reset()
             model.save('mnist_checkpoints/{:02d}'.format(e))
 
 

model.py

@@ -26,7 +26,7 @@ from paddle.fluid.framework import in_dygraph_mode, Variable
 from paddle.fluid.executor import global_scope
 from paddle.fluid.io import is_belong_to_optimizer
 from paddle.fluid.dygraph.base import to_variable
-from metrics.metric import Metric
+from metrics import Metric
 
 __all__ = ['shape_hints', 'Model', 'Loss', 'CrossEntropy']
 
@@ -297,6 +297,8 @@ class StaticGraphAdapter(object):
                 metric.update(outputs, labels)
             return outputs, losses
         else: # train
+            for metric in self.model._metrics:
+                metric.update(outputs, labels)
             return outputs, losses
 
     def _make_program(self, inputs):
@@ -311,7 +313,7 @@ class StaticGraphAdapter(object):
             if self.mode != 'test':
                 label_vars = self._infer_label_vars(outputs)
                 self._label_vars[self.mode] = label_vars
-                losses = self.model._loss_function(outputs[0], label_vars)
+                losses = self.model._loss_function(outputs, label_vars)
                 if self.mode == 'train':
                     self._loss_endpoint = fluid.layers.sum(losses)
                     self.model._optimizer.minimize(self._loss_endpoint)
@@ -319,7 +321,7 @@ class StaticGraphAdapter(object):
             prog = prog.clone(for_test=True)
         self._progs[self.mode] = prog
         self._endpoints[self.mode] = {
-            "output": outputs[1:],
+            "output": outputs,
             "label": label_vars,
             "loss": losses,
         }
@@ -419,12 +421,14 @@ class DynamicGraphAdapter(object):
         self.mode = 'train'
         inputs = to_list(inputs)
         labels = to_list(labels)
-        outputs = self.model.forward(*[to_variable(x) for x in inputs])[0]
+        outputs = self.model.forward(*[to_variable(x) for x in inputs])
         losses = self.model._loss_function(outputs, labels)
         final_loss = fluid.layers.sum(losses)
         final_loss.backward()
         self.model._optimizer.minimize(final_loss)
         self.model.clear_gradients()
+        for metric in self.model._metrics:
+            metric.update([to_numpy(o) for o in to_list(outputs)], labels)
         return [to_numpy(o) for o in to_list(outputs)], \
             [to_numpy(l) for l in losses]
 
@@ -436,18 +440,18 @@ class DynamicGraphAdapter(object):
         inputs = to_list(inputs)
         labels = to_list(labels)
         outputs = self.model.forward(*[to_variable(x) for x in inputs])
-        losses = self.model._loss_function(outputs[0], labels)
+        losses = self.model._loss_function(outputs, labels)
         for metric in self.model._metrics:
-            metric.update([to_numpy(o) for o in outputs[1:]], labels)
-        return [to_numpy(o) for o in to_list(outputs[0])], \
+            metric.update([to_numpy(o) for o in to_list(outputs)], labels)
+        return [to_numpy(o) for o in to_list(outputs)], \
             [to_numpy(l) for l in losses]
 
     def test(self, inputs, device='CPU', device_ids=None):
         super(Model, self.model).eval()
         self.mode = 'test'
         inputs = [to_variable(x) for x in to_list(inputs)]
-        outputs = self.model.forward(*inputs)[1:]
-        return [to_numpy(o) for o in to_list(outputs[1:])]
+        outputs = self.model.forward(*inputs)
+        return [to_numpy(o) for o in to_list(outputs)]
 
     def parameters(self, *args, **kwargs):
         return super(Model, self.model).parameters(*args, **kwargs)

yolov3.py

@@ -35,7 +35,6 @@ from paddle.fluid.regularizer import L2Decay
 
 from model import Model, Loss, shape_hints
 from resnet import ResNet, ConvBNLayer
-from metrics.coco import COCOMetric
 
 import logging
 FORMAT = '%(asctime)s-%(levelname)s: %(message)s'
@@ -174,7 +173,7 @@ class YOLOv3(Model):
                 route = self.route_blocks[idx](route)
                 route = fluid.layers.resize_nearest(route, scale=2)
 
-            if self.mode != 'train':
+            if self.mode == 'test':
                 anchor_mask = self.anchor_masks[idx]
                 mask_anchors = []
                 for m in anchor_mask:
@@ -195,10 +194,10 @@ class YOLOv3(Model):
 
             downsample //= 2
 
-        if self.mode == 'train':
-            return [outputs]
+        if self.mode != 'test':
+            return outputs
 
-        return [outputs, img_id, fluid.layers.multiclass_nms(
+        return [img_id, fluid.layers.multiclass_nms(
             bboxes=fluid.layers.concat(boxes, axis=1),
             scores=fluid.layers.concat(scores, axis=2),
             score_threshold=self.valid_thresh,
@@ -533,12 +532,15 @@ def main():
         anno_path = os.path.join(FLAGS.data, 'annotations', 'instances_val2017.json')
         model.prepare(optim,
                       YoloLoss(num_classes=NUM_CLASSES),
-                      metrics=COCOMetric(anno_path, with_background=False))
+                      # For YOLOv3, output variable in train/eval is different,
+                      # which is not supported by metric, add by callback later?
+                      # metrics=COCOMetric(anno_path, with_background=False)
+                      )
 
         for e in range(epoch):
-            # logger.info("======== train epoch {} ========".format(e))
-            # run(model, train_loader)
-            # model.save('yolo_checkpoints/{:02d}'.format(e))
+            logger.info("======== train epoch {} ========".format(e))
+            run(model, train_loader)
+            model.save('yolo_checkpoints/{:02d}'.format(e))
             logger.info("======== eval epoch {} ========".format(e))
             run(model, val_loader, mode='eval')
             # should be called in fit()

metrics/__init__.py → yolov3/__init__.py

@@ -12,5 +12,3 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import absolute_import
-

metrics/coco.py → yolov3/coco_metric.py

@@ -12,14 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import absolute_import
-
 import sys
 import json
 from pycocotools.cocoeval import COCOeval
 from pycocotools.coco import COCO
 
-from .metric import Metric
+from metrics import Metric
 
 import logging
 FORMAT = '%(asctime)s-%(levelname)s: %(message)s'
@@ -32,6 +30,7 @@ __all__ = ['COCOMetric']
 OUTFILE = './bbox.json'
 
 
+# considered to change to a callback later
 class COCOMetric(Metric):
     """
     Metrci for MS-COCO dataset, only support update with batch