Faster RCNN change epoch to iter (#1254)

* change epoch to iter
* add smooth loss for logging

fluid/faster_rcnn/reader.py

@@ -22,6 +22,7 @@ import os
 import time
 import copy
 import six
+from collections import deque
 
 from roidbs import JsonDataset
 import data_utils
@@ -67,28 +68,47 @@ def coco(settings, mode, batch_size=None, shuffle=False):
 
     print("{} on {} with {} roidbs".format(mode, settings.dataset, len(roidbs)))
 
-    def reader():
-        if mode == "train" and shuffle:
-            random.shuffle(roidbs)
-        batch_out = []
-        for roidb in roidbs:
-            im, im_scales = data_utils.get_image_blob(roidb, settings)
-            im_id = roidb['id']
+    def roidb_reader(roidb):
+        im, im_scales = data_utils.get_image_blob(roidb, settings)
+        im_id = roidb['id']
+        im_height = np.round(roidb['height'] * im_scales)
+        im_width = np.round(roidb['width'] * im_scales)
+        im_info = np.array([im_height, im_width, im_scales], dtype=np.float32)
+        gt_boxes = roidb['gt_boxes'].astype('float32')
+        gt_classes = roidb['gt_classes'].astype('int32')
+        is_crowd = roidb['is_crowd'].astype('int32')
+        return im, gt_boxes, gt_classes, is_crowd, im_info, im_id
 
-            im_height = np.round(roidb['height'] * im_scales)
-            im_width = np.round(roidb['width'] * im_scales)
-            im_info = np.array(
-                [im_height, im_width, im_scales], dtype=np.float32)
-            gt_boxes = roidb['gt_boxes'].astype('float32')
-            gt_classes = roidb['gt_classes'].astype('int32')
-            is_crowd = roidb['is_crowd'].astype('int32')
-            if mode == 'train' and gt_boxes.shape[0] == 0:
-                continue
-            batch_out.append(
-                (im, gt_boxes, gt_classes, is_crowd, im_info, im_id))
-            if len(batch_out) == batch_size:
-                yield batch_out
-                batch_out = []
+    def reader():
+        if mode == "train":
+            roidb_perm = deque(np.random.permutation(roidbs))
+            roidb_cur = 0
+            batch_out = []
+            while True:
+                roidb = roidb_perm[0]
+                roidb_cur += 1
+                roidb_perm.rotate(-1)
+                if roidb_cur >= len(roidbs):
+                    roidb_perm = deque(np.random.permutation(roidbs))
+                im, gt_boxes, gt_classes, is_crowd, im_info, im_id = roidb_reader(
+                    roidb)
+                if gt_boxes.shape[0] == 0:
+                    continue
+                batch_out.append(
+                    (im, gt_boxes, gt_classes, is_crowd, im_info, im_id))
+                if len(batch_out) == batch_size:
+                    yield batch_out
+                    batch_out = []
+        else:
+            batch_out = []
+            for roidb in roidbs:
+                im, gt_boxes, gt_classes, is_crowd, im_info, im_id = roidb_reader(
+                    roidb)
+                batch_out.append(
+                    (im, gt_boxes, gt_classes, is_crowd, im_info, im_id))
+                if len(batch_out) == batch_size:
+                    yield batch_out
+                    batch_out = []
 
     return reader
 

fluid/faster_rcnn/roidbs.py

@@ -45,7 +45,7 @@ class JsonDataset(object):
     """A class representing a COCO json dataset."""
 
     def __init__(self, args, train=False):
-        logger.debug('Creating: {}'.format(args.dataset))
+        print('Creating: {}'.format(args.dataset))
         self.name = args.dataset
         self.is_train = train
         if self.is_train:
@@ -89,13 +89,14 @@ class JsonDataset(object):
             for entry in roidb:
                 self._add_gt_annotations(entry)
             end_time = time.time()
-            logger.debug('_add_gt_annotations took {:.3f}s'.format(end_time -
-                                                                   start_time))
+            print('_add_gt_annotations took {:.3f}s'.format(end_time -
+                                                            start_time))
 
-            logger.info('Appending horizontally-flipped training examples...')
+            print('Appending horizontally-flipped training examples...')
             self._extend_with_flipped_entries(roidb)
-        logger.info('Loaded dataset: {:s}'.format(self.name))
-        logger.info('{:d} roidb entries'.format(len(roidb)))
+        print('Loaded dataset: {:s}'.format(self.name))
+        print('{:d} roidb entries'.format(len(roidb)))
+        self._filter_for_training(roidb)
         return roidb
 
     def _prep_roidb_entry(self, entry):
@@ -182,3 +183,22 @@ class JsonDataset(object):
             flipped_entry['flipped'] = True
             flipped_roidb.append(flipped_entry)
         roidb.extend(flipped_roidb)
+
+    def _filter_for_training(self, roidb):
+        """Remove roidb entries that have no usable RoIs based on config settings.
+        """
+
+        def is_valid(entry):
+            # Valid images have:
+            #   (1) At least one groundtruth RoI OR
+            #   (2) At least one background RoI
+            gt_boxes = entry['gt_boxes']
+            # image is only valid if such boxes exist
+            valid = len(gt_boxes) > 0
+            return valid
+
+        num = len(roidb)
+        filtered_roidb = [entry for entry in roidb if is_valid(entry)]
+        num_after = len(filtered_roidb)
+        print('Filtered {} roidb entries: {} -> {}'.format(num - num_after, num,
+                                                           num_after))

fluid/faster_rcnn/train.py

@@ -5,7 +5,7 @@ import argparse
 import functools
 import shutil
 import cPickle
-from utility import add_arguments, print_arguments
+from utility import add_arguments, print_arguments, SmoothedValue
 
 import paddle
 import paddle.fluid as fluid
@@ -24,11 +24,13 @@ add_arg('model_save_dir',   str,    'output',     "The path to save model.")
 add_arg('pretrained_model', str,    'imagenet_resnet50_fusebn', "The init model path.")
 add_arg('dataset',          str,    'coco2017', "coco2014, coco2017, and pascalvoc.")
 add_arg('data_dir',         str,    'data/COCO17', "data directory")
-add_arg('class_num',        int,   81,          "Class number.")
-add_arg('use_pyreader',     bool,   True,          "Class number.")
+add_arg('class_num',        int,    81,             "Class number.")
+add_arg('use_pyreader',     bool,   True,           "Use pyreader.")
 # SOLVER
 add_arg('learning_rate',    float,  0.01,     "Learning rate.")
-add_arg('num_passes',       int,    20,        "Epoch number.")
+add_arg('max_iter',         int,    180000,   "Iter number.")
+add_arg('log_window',       int,    1,        "Log smooth window, set 1 for debug, set 20 for train.")
+add_arg('snapshot_stride',  int,    10000,    "save model every snapshot stride.")
 # RPN
 add_arg('anchor_sizes',     int,    [32,64,128,256,512],  "The size of anchors.")
 add_arg('aspect_ratios',    float,  [0.5,1.0,2.0],    "The ratio of anchors.")
@@ -45,8 +47,6 @@ add_arg('debug',            bool,   False,   "Debug mode")
 #yapf: enable
 
 def train(cfg):
-    num_passes = cfg.num_passes
-    batch_size = cfg.batch_size
     learning_rate = cfg.learning_rate
     image_shape = [3, cfg.max_size, cfg.max_size]
 
@@ -121,53 +121,58 @@ def train(cfg):
 
     fetch_list = [loss, rpn_cls_loss, rpn_reg_loss, loss_cls, loss_bbox]
 
-    def train_step_pyreader(epoc_id):
+    def train_step_pyreader():
         py_reader.start()
+        smoothed_loss = SmoothedValue(cfg.log_window)
         try:
             start_time = time.time()
             prev_start_time = start_time
             every_pass_loss = []
-            batch_id = 0
-            while True:
+            for iter_id in range(cfg.max_iter):
                 prev_start_time = start_time
                 start_time = time.time()
                 losses = train_exe.run(fetch_list=[v.name for v in fetch_list])
                 every_pass_loss.append(np.mean(np.array(losses[0])))
+                smoothed_loss.add_value(np.mean(np.array(losses[0])))
                 lr = np.array(fluid.global_scope().find_var('learning_rate').get_tensor())
-                print("Epoc {:d}, batch {:d}, lr {:.6f}, loss {:.6f}, time {:.5f}".format(
-                      epoc_id, batch_id, lr[0], losses[0][0], start_time - prev_start_time))
-                batch_id += 1
+                print("Iter {:d}, lr {:.6f}, loss {:.6f}, time {:.5f}".format(
+                      iter_id, lr[0], smoothed_loss.get_median_value(), start_time - prev_start_time))
                 #print('cls_loss ', losses[1][0], ' reg_loss ', losses[2][0], ' loss_cls ', losses[3][0], ' loss_bbox ', losses[4][0])
+                if (iter_id + 1) % cfg.snapshot_stride == 0:
+                    save_model("model_iter{}".format(iter_id))
         except fluid.core.EOFException:
             py_reader.reset()
         return np.mean(every_pass_loss)
 
-    def train_step(epoc_id):
+    def train_step():
         start_time = time.time()
         prev_start_time = start_time
         start = start_time
         every_pass_loss = []
-        for batch_id, data in enumerate(train_reader()):
+        smoothed_loss = SmoothedValue(cfg.log_window)
+        for iter_id, data in enumerate(train_reader()):
             prev_start_time = start_time
             start_time = time.time()
             losses = train_exe.run(fetch_list=[v.name for v in fetch_list],
                                    feed=feeder.feed(data))
             loss_v = np.mean(np.array(losses[0]))
             every_pass_loss.append(loss_v)
-
+            smoothed_loss.add_value(loss_v)
             lr = np.array(fluid.global_scope().find_var('learning_rate').get_tensor())
-            print("Epoc {:d}, batch {:d}, lr {:.6f}, loss {:.6f}, time {:.5f}".format(
-                  epoc_id, batch_id, lr[0], losses[0][0], start_time - prev_start_time))
+            print("Iter {:d}, lr {:.6f}, loss {:.6f}, time {:.5f}".format(
+                  iter_id, lr[0], smoothed_loss.get_median_value(), start_time - prev_start_time))
             #print('cls_loss ', losses[1][0], ' reg_loss ', losses[2][0], ' loss_cls ', losses[3][0], ' loss_bbox ', losses[4][0])
+            if (iter_id + 1) % cfg.snapshot_stride == 0:
+                save_model("model_iter{}".format(iter_id))
+            if (iter_id + 1) == cfg.max_iter:
+                break
         return np.mean(every_pass_loss)
 
-
-    for epoc_id in range(num_passes):
-        if cfg.use_pyreader:
-            train_step_pyreader(epoc_id)
-        else:
-            train_step(epoc_id)
-        save_model(str(epoc_id))
+    if cfg.use_pyreader:
+        train_step_pyreader()
+    else:
+        train_step()
+    save_model('model_final')
 
 if __name__ == '__main__':
     args = parser.parse_args()

fluid/faster_rcnn/utility.py

@@ -20,6 +20,7 @@ from __future__ import print_function
 import distutils.util
 import numpy as np
 import six
+from collections import deque
 from paddle.fluid import core
 
 
@@ -62,3 +63,18 @@ def add_arguments(argname, type, default, help, argparser, **kwargs):
         type=type,
         help=help + ' Default: %(default)s.',
         **kwargs)
+
+
+class SmoothedValue(object):
+    """Track a series of values and provide access to smoothed values over a
+    window or the global series average.
+    """
+
+    def __init__(self, window_size):
+        self.deque = deque(maxlen=window_size)
+
+    def add_value(self, value):
+        self.deque.append(value)
+
+    def get_median_value(self):
+        return np.median(self.deque)