refine mixup score and label smooth

fluid/PaddleCV/yolov3/box_utils.py

@@ -123,16 +123,15 @@ def box_crop(boxes, labels, scores, crop, img_shape):
     boxes[:, 1], boxes[:, 3] = (boxes[:, 1] - boxes[:, 3] / 2) * im_h, (boxes[:, 1] + boxes[:, 3] / 2) * im_h
 
     crop_box = np.array([x, y, x + w, y + h])
-    # centers = (boxes[:, :2] + boxes[:, 2:]) / 2.0
-    # mask = np.logical_and(crop_box[:2] <= centers, centers <= crop_box[2:]).all(axis=1)
+    centers = (boxes[:, :2] + boxes[:, 2:]) / 2.0
+    mask = np.logical_and(crop_box[:2] <= centers, centers <= crop_box[2:]).all(axis=1)
 
     boxes[:, :2] = np.maximum(boxes[:, :2], crop_box[:2])
     boxes[:, 2:] = np.minimum(boxes[:, 2:], crop_box[2:])
     boxes[:, :2] -= crop_box[:2]
     boxes[:, 2:] -= crop_box[:2]
 
-    # mask = np.logical_and(mask, (boxes[:, :2] < boxes[:, 2:]).all(axis=1))
-    mask =  (boxes[:, :2] < boxes[:, 2:]).all(axis=1)
+    mask = np.logical_and(mask, (boxes[:, :2] < boxes[:, 2:]).all(axis=1))
     boxes = boxes * np.expand_dims(mask.astype('float32'), axis=1)
     labels = labels * mask.astype('float32')
     scores = scores * mask.astype('float32')

fluid/PaddleCV/yolov3/data_utils.py

@@ -73,12 +73,11 @@ class GeneratorEnqueuer(object):
                             size = self.random_sizes[queue_idx]
                             for g in generator_output:
                                 g[0] = g[0].transpose((1, 2, 0))
-                                g[0] = image_utils.random_interp(g[0], size, cv2.INTER_LINEAR)
+                                g[0] = image_utils.random_interp(g[0], size)
                                 g[0] = g[0].transpose((2, 0, 1))
                             try:
                                 self.queues[queue_idx].put_nowait(generator_output)
                             except:
-                                timw.sleep(self.wait_time)
                                 continue
                             else:
                                 break

fluid/PaddleCV/yolov3/infer.py

@@ -35,7 +35,6 @@ def infer():
     # yapf: enable
     feeder = fluid.DataFeeder(place=place, feed_list=model.feeds())
     fetch_list = [outputs]
-    # fetch_list = outputs
     image_names = []
     if cfg.image_name is not None:
         image_names.append(cfg.image_name)
@@ -55,6 +54,7 @@ def infer():
         bboxes = np.array(outputs[0])
         if bboxes.shape[1] != 6:
             print("No object found in {}".format(image_name))
+            continue
         labels = bboxes[:, 0].astype('int32')
         scores = bboxes[:, 1].astype('float32')
         boxes = bboxes[:, 2:].astype('float32')

fluid/PaddleCV/yolov3/models.py

@@ -206,13 +206,13 @@ class YOLOv3(object):
                             x=out,
                             gtbox=self.gtbox,
                             gtlabel=self.gtlabel,
-                            # gtscore=self.gtscore,
+                            gtscore=self.gtscore,
                             anchors=anchors,
                             anchor_mask=anchor_mask,
                             class_num=class_num,
                             ignore_thresh=ignore_thresh,
                             downsample_ratio=self.downsample,
-                            # use_label_smooth=False,
+                            use_label_smooth=cfg.label_smooth,
                             name="yolo_loss"+str(i))
                     self.losses.append(fluid.layers.reduce_mean(loss))
                 else:

fluid/PaddleCV/yolov3/reader.py

@@ -38,10 +38,6 @@ class DataSetReader(object):
         self.has_parsed_categpry = False
 
     def _parse_dataset_dir(self, mode):
-        # cfg.data_dir = "dataset/coco"
-        # cfg.train_file_list = 'annotations/instances_val2017.json'
-        # cfg.train_data_dir = 'val2017'
-        # cfg.dataset = "coco2017"
         if 'coco2014' in cfg.dataset:
             cfg.train_file_list = 'annotations/instances_train2014.json'
             cfg.train_data_dir = 'train2014'
@@ -115,7 +111,6 @@ class DataSetReader(object):
         image_ids = self.COCO.getImgIds()
         image_ids.sort()
         imgs = copy.deepcopy(self.COCO.loadImgs(image_ids))
-        # imgs = imgs[-8:]
         for img in imgs:
             img['image'] = os.path.join(self.img_dir, img['file_name'])
             assert os.path.exists(img['image']), \
@@ -141,7 +136,7 @@ class DataSetReader(object):
         else:
             return self._parse_images(is_train=(mode=='train'))
 
-    def get_reader(self, mode, size=416, batch_size=None, shuffle=False, random_shape_iter=0, random_sizes=[], image=None):
+    def get_reader(self, mode, size=416, batch_size=None, shuffle=False, mixup_iter=0, random_sizes=[], image=None):
         assert mode in ['train', 'test', 'infer'], "Unknow mode type!"
         if mode != 'infer':
             assert batch_size is not None, "batch size connot be None in mode {}".format(mode)
@@ -172,6 +167,16 @@ class DataSetReader(object):
             gt_labels = img['gt_labels'].copy()
             gt_scores = np.ones_like(gt_labels)
 
+            if mixup_img:
+                mixup_im = cv2.imread(mixup_img['image'])
+                mixup_im = cv2.cvtColor(mixup_im, cv2.COLOR_BGR2RGB)
+                mixup_gt_boxes = np.array(mixup_img['gt_boxes']).copy()
+                mixup_gt_labels = np.array(mixup_img['gt_labels']).copy()
+                mixup_gt_scores = np.ones_like(mixup_gt_labels)
+                im, gt_boxes, gt_labels, gt_scores = image_utils.image_mixup(im, gt_boxes, \
+                        gt_labels, gt_scores, mixup_im, mixup_gt_boxes, mixup_gt_labels, \
+                        mixup_gt_scores)
+
             im, gt_boxes, gt_labels, gt_scores = image_utils.image_augment(im, gt_boxes, gt_labels, gt_scores, size, mean)
 
             mean = np.array(mean).reshape((1, 1, -1))
@@ -186,6 +191,14 @@ class DataSetReader(object):
                 return np.random.choice(random_sizes)
             return size
 
+        def get_mixup_img(imgs, mixup_iter, total_iter, read_cnt):
+            if total_iter >= mixup_iter:
+                return None
+
+            mixup_idx = np.random.randint(1, len(imgs))
+            mixup_img = imgs[(read_cnt + mixup_idx) % len(imgs)]
+            return mixup_img
+
         def reader():
             if mode == 'train':
                 imgs = self._parse_images_by_mode(mode)
@@ -195,25 +208,21 @@ class DataSetReader(object):
                 total_iter = 0
                 batch_out = []
                 img_size = get_img_size(size, random_sizes)
-                # img_ids = []
                 while True:
                     img = imgs[read_cnt % len(imgs)]
-		    mixup_img = None
+                    mixup_img = get_mixup_img(imgs, mixup_iter, total_iter, read_cnt)
                     read_cnt += 1
                     if read_cnt % len(imgs) == 0 and shuffle:
                         np.random.shuffle(imgs)
                     im, gt_boxes, gt_labels, gt_scores = img_reader_with_augment(img, img_size, cfg.pixel_means, cfg.pixel_stds, mixup_img)
                     batch_out.append([im, gt_boxes, gt_labels, gt_scores])
-                    # img_ids.append((img['id'], mixup_img['id'] if mixup_img else -1))
 
                     if len(batch_out) == batch_size:
-                        # print("img_ids: ", img_ids)
                         yield batch_out
                         batch_out = []
                         total_iter += 1
                         if total_iter % 10 == 0:
                             img_size = get_img_size(size, random_sizes)
-                        # img_ids = []
 
             elif mode == 'test':
                 imgs = self._parse_images_by_mode(mode)
@@ -242,14 +251,14 @@ dsr = DataSetReader()
 def train(size=416, 
           batch_size=64, 
           shuffle=True, 
-          random_shape_iter=0,
+          mixup_iter=0,
           random_sizes=[],
           interval=10,
           pyreader_num=1,
           num_workers=16,
           max_queue=32,
           use_multiprocessing=True):
-    generator = dsr.get_reader('train', size, batch_size, shuffle, random_shape_iter, random_sizes)
+    generator = dsr.get_reader('train', size, batch_size, shuffle, int(mixup_iter/pyreader_num), random_sizes)
 
     if not use_multiprocessing:
         return generator
@@ -267,7 +276,6 @@ def train(size=416,
             generator_out = None
             np.random.seed(1000)
             intervals = pyreader_num * interval
-	    total_random_iter = pyreader_num * random_shape_iter
             cnt = 0
 	    idx = len(random_sizes) - 1
             while True:
@@ -282,9 +290,7 @@ def train(size=416,
                 cnt += 1
                 if cnt % intervals == 0:
                     idx = np.random.randint(len(random_sizes))
-		    if cnt >= total_random_iter:
-			idx = -1
-		    print("Resizing: ", random_sizes[idx])
+                    print("Resizing: ", random_sizes[idx])
         finally:
             if enqueuer is not None:
                 enqueuer.stop()

fluid/PaddleCV/yolov3/train.py

@@ -94,13 +94,13 @@ def train():
     if cfg.random_shape:
         random_sizes = [32 * i for i in range(10, 20)]
 
-    random_shape_iter = cfg.max_iter - cfg.start_iter - cfg.tune_iter
+    mixup_iter = cfg.max_iter - cfg.start_iter - cfg.no_mixup_iter
     if cfg.use_pyreader:
-        train_reader = reader.train(input_size, batch_size=int(hyperparams['batch'])/devices_num, shuffle=True, random_shape_iter=random_shape_iter, random_sizes=random_sizes, interval=10, pyreader_num=devices_num,use_multiprocessing=cfg.use_multiprocess)
+        train_reader = reader.train(input_size, batch_size=int(hyperparams['batch'])/devices_num, shuffle=True, mixup_iter=mixup_iter*devices_num, random_sizes=random_sizes, interval=10, pyreader_num=devices_num, use_multiprocessing=cfg.use_multiprocess)
         py_reader = model.py_reader
         py_reader.decorate_paddle_reader(train_reader)
     else:
-        train_reader = reader.train(input_size, batch_size=int(hyperparams['batch']), shuffle=True, random_shape_iter=random_shape_iter, random_sizes=random_sizes, interval=10,use_multiprocessing=cfg.use_multiprocess)
+        train_reader = reader.train(input_size, batch_size=int(hyperparams['batch']), shuffle=True, mixup_iter=mixup_iter, random_sizes=random_sizes, use_multiprocessing=cfg.use_multiprocess)
         feeder = fluid.DataFeeder(place=place, feed_list=model.feeds())
 
     def save_model(postfix):
@@ -150,7 +150,6 @@ def train():
         snapshot_loss = 0
         snapshot_time = 0
         for iter_id, data in enumerate(train_reader()):
-	    print(len(data), data[0][0].shape)
             iter_id += cfg.start_iter
             prev_start_time = start_time
             start_time = time.time()

fluid/PaddleCV/yolov3/utility.py

@@ -114,8 +114,9 @@ def parse_args():
     # TRAIN TEST INFER
     add_arg('input_size',       int,    608,    "Image input size of YOLOv3.")
     add_arg('random_shape',     bool,   True,     "Resize to random shape for train reader.")
-    add_arg('tune_iter',	int,    200,      "Disable random shape in last N iter.")
-    add_arg('valid_thresh',     float,  0.005,    "Valid confidence score for NMS.")
+    add_arg('label_smooth',     bool,   True,     "Use label smooth in class label.")
+    add_arg('no_mixup_iter',    int,    40000,      "Disable mixup in last N iter.")
+    add_arg('valid_thresh',     float,  0.01,    "Valid confidence score for NMS.")
     add_arg('nms_thresh',       float,  0.45,    "NMS threshold.")
     add_arg('nms_topk',         int,    400,    "The number of boxes to perform NMS.")
     add_arg('nms_posk',         int,    100,    "The number of boxes of NMS output.")