refine FPN code

fluid/PaddleCV/rcnn/config.py

@@ -55,7 +55,7 @@ _C.TRAIN.padding_minibatch = False
 _C.TRAIN.snapshot_iter = 10000
 
 # number of RPN proposals to keep before NMS
-_C.TRAIN.rpn_pre_nms_top_n = 12000
+_C.TRAIN.rpn_pre_nms_top_n = 2000
 
 # number of RPN proposals to keep after NMS
 _C.TRAIN.rpn_post_nms_top_n = 2000
@@ -208,8 +208,8 @@ _C.FPN_rpn_aspect_ratios = (0.5, 1, 2)
 _C.FPN_rpn_anchor_start_size = 32
 
 # Parameters to map RoI level
-_C.FPN_roi_canonical_level = 224
-_C.FPN_roi_canonical_scale = 4
+_C.FPN_roi_canonical_level = 4
+_C.FPN_roi_canonical_scale = 224
 
 # Stride of the coarsest FPN level
 _C.FPN_coarsest_stride = 32

fluid/PaddleCV/rcnn/models/FPN.py

@@ -57,6 +57,7 @@ def add_fpn_onto_conv_body(res_dict, res_name_list):
             fpn_dim, )
     fpn_dict = {}
     fpn_name_list = []
+
     for i in range(num_backbone_stages):
         fpn_name = 'fpn_' + res_name_list[i]
         fpn_output = fluid.layers.conv2d(
@@ -88,7 +89,7 @@ def add_fpn_onto_conv_body(res_dict, res_name_list):
 
 def add_topdown_lateral_module(index, res_dict, res_name_list, fpn_inner_output,
                                fpn_dim):
-    lateral_name = 'fpn_lateral_' + res_name_list[index]
+    lateral_name = 'fpn_inner_' + res_name_list[index] + '_lateral'
     topdown_name = 'fpn_topdown_' + res_name_list[index]
     fpn_inner_name = 'fpn_inner_' + res_name_list[index]
     lateral = fluid.layers.conv2d(
@@ -130,6 +131,7 @@ def add_fpn_rpn_outputs(fpn_dict, im_info, fpn_name_list, mode):
     rpn_rois_list = []
     rpn_roi_probs_list = []
     anchors_list = []
+    anchor_num_list = []
     var_list = []
     for lvl in range(k_min, k_max + 1):
         input_name = fpn_name_list[k_max - lvl]
@@ -152,6 +154,7 @@ def add_fpn_rpn_outputs(fpn_dict, im_info, fpn_name_list, mode):
                 name=conv_share_name + '_b',
                 learning_rate=2.,
                 regularizer=L2Decay(0.)))
+        conv_rpn_fpn.persistable = True
         rpn_cls_logits_fpn = fluid.layers.conv2d(
             input=conv_rpn_fpn,
             num_filters=num_anchors,
@@ -166,6 +169,15 @@ def add_fpn_rpn_outputs(fpn_dict, im_info, fpn_name_list, mode):
                 name=cls_share_name + '_b',
                 learning_rate=2.,
                 regularizer=L2Decay(0.)))
+        shape = fluid.layers.shape(rpn_cls_logits_fpn)
+        shape_chw = fluid.layers.slice(shape, axes=[0], starts=[1], ends=[4])
+        anchors_num = fluid.layers.reduce_prod(shape_chw)
+        if lvl == k_min:
+            anchor_num_list.append(anchors_num)
+        else:
+            anchor_num_list.append(anchor_num_list[-1] + anchors_num)
+        anchor_num_list[-1].stop_gradient = True
+        rpn_cls_logits_fpn.persistable = True
         rpn_bbox_pred_fpn = fluid.layers.conv2d(
             input=conv_rpn_fpn,
             num_filters=num_anchors * 4,
@@ -180,6 +192,7 @@ def add_fpn_rpn_outputs(fpn_dict, im_info, fpn_name_list, mode):
                 name=bbox_share_name + '_b',
                 learning_rate=2.,
                 regularizer=L2Decay(0.)))
+        rpn_bbox_pred_fpn.persistable = True
         rpn_fpn_list.append((rpn_cls_logits_fpn, rpn_bbox_pred_fpn))
 
         anchors, var = fluid.layers.anchor_generator(
@@ -188,14 +201,18 @@ def add_fpn_rpn_outputs(fpn_dict, im_info, fpn_name_list, mode):
             aspect_ratios=cfg.FPN_rpn_aspect_ratios,
             variance=cfg.variances,
             stride=(2.**lvl, 2.**lvl))
+
         rpn_cls_probs_fpn = fluid.layers.sigmoid(
             rpn_cls_logits_fpn, name='rpn_cls_probs_fpn' + slvl)
+        rpn_cls_probs_fpn.persistable = True
+
         param_obj = cfg.TRAIN if mode == 'train' else cfg.TEST
         pre_nms_top_n = param_obj.rpn_pre_nms_top_n
         post_nms_top_n = param_obj.rpn_post_nms_top_n
         nms_thresh = param_obj.rpn_nms_thresh
         min_size = param_obj.rpn_min_size
         eta = param_obj.rpn_eta
+
         rpn_rois_fpn, rpn_roi_probs_fpn = fluid.layers.generate_proposals(
             scores=rpn_cls_probs_fpn,
             bbox_deltas=rpn_bbox_pred_fpn,
@@ -207,11 +224,13 @@ def add_fpn_rpn_outputs(fpn_dict, im_info, fpn_name_list, mode):
             nms_thresh=nms_thresh,
             min_size=min_size,
             eta=eta)
+        rpn_rois_fpn.persistable = True
+        rpn_roi_probs_fpn.persistable = True
         rpn_rois_list.append(rpn_rois_fpn)
         rpn_roi_probs_list.append(rpn_roi_probs_fpn)
         anchors_list.append(anchors)
         var_list.append(var)
-    return rpn_fpn_list, rpn_rois_list, rpn_roi_probs_list, anchors_list, var_list
+    return rpn_fpn_list, rpn_rois_list, rpn_roi_probs_list, anchors_list, var_list, anchor_num_list
 
 
 def add_FPN_roi_head(head_inputs, rois_list, fpn_name_list, spatial_scale):
@@ -219,9 +238,10 @@ def add_FPN_roi_head(head_inputs, rois_list, fpn_name_list, spatial_scale):
     k_min = cfg.FPN_roi_min_level
     num_roi_lvls = k_max - k_min + 1
     input_name_list = fpn_name_list[-num_roi_lvls:]
+    spatial_scale = spatial_scale[-num_roi_lvls:]
     roi_out_list = []
     for lvl in range(k_min, k_max + 1):
-        rois = rois_list[k_max - lvl]
+        rois = rois_list[lvl - k_min]
         input_name = input_name_list[k_max - lvl]
         head_input = head_inputs[input_name]
         sc = spatial_scale[k_max - lvl]
@@ -243,6 +263,7 @@ def add_FPN_roi_head(head_inputs, rois_list, fpn_name_list, spatial_scale):
                 sampling_ratio=cfg.sampling_ratio,
                 name='roi_align_lvl_' + str(lvl))
         roi_out_list.append(roi_out)
+        roi_out.persistable = True
 
     return roi_out_list
 
@@ -257,6 +278,7 @@ def add_FPN_roi_head_output(body_dict, pool_rois, body_name_list,
     roi_feat_shuffle = fluid.layers.concat(roi_out_list)
     roi_feat = fluid.layers.gather(roi_feat_shuffle, restore_index)
     roi_feat = fluid.layers.lod_reset(roi_feat, rois_collect)
+    roi_feat.persistable = True
     fc6 = fluid.layers.fc(input=roi_feat,
                           size=cfg.MLP_HEAD_DIM,
                           act='relu',
@@ -275,4 +297,5 @@ def add_FPN_roi_head_output(body_dict, pool_rois, body_name_list,
                               name='fc7_b',
                               learning_rate=2.,
                               regularizer=L2Decay(0.)))
+    fc7.persistable = True
     return roi_feat, fc7

fluid/PaddleCV/rcnn/models/model_builder.py

@@ -43,8 +43,8 @@ class RCNN(object):
         if cfg.FPN_ON:
             body_dict, self.spatial_scale, body_name_list = FPN.add_fpn_onto_conv_body(
                 body_dict, body_name_list)
+
         # RPN
-        #print(body_dict)
         self.rpn_heads(body_dict, body_name_list)
         # Fast RCNN
         self.fast_rcnn_heads(body_dict, body_name_list)
@@ -188,21 +188,18 @@ class RCNN(object):
         self.rpn_roi_probs_list = fpn_outputs[2]
         self.anchors_list = fpn_outputs[3]
         self.var_list = fpn_outputs[4]
+        self.anchor_num_list = fpn_outputs[5]
         param_obj = cfg.TRAIN if self.mode == 'train' else cfg.TEST
         post_nms_top_n = param_obj.rpn_post_nms_top_n
         rois_collect = fluid.layers.collect_fpn_proposals(
             self.rpn_rois_list,
             self.rpn_roi_probs_list,
-            cfg.FPN_rpn_max_level,
             cfg.FPN_rpn_min_level,
+            cfg.FPN_rpn_max_level,
             post_nms_top_n,
             name='collect')
-        rois_collect.persistable = True
-        fluid.layers.Print(rois_collect)
         if self.mode == 'train':
             rois_collect = self.generate_labels(rois_collect)
-        #rois_collect.persistable = True
-        #fluid.layers.Print(rois_collect)
         rois, self.restore_index = fluid.layers.distribute_fpn_proposals(
             rois_collect,
             cfg.FPN_roi_min_level,
@@ -210,6 +207,10 @@ class RCNN(object):
             cfg.FPN_roi_canonical_level,
             cfg.FPN_roi_canonical_scale,
             name='distribute')
+
+        for roi in rois:
+            roi.persistable = True
+        self.restore_index.persistable = True
         return rois_collect, rois, self.restore_index
 
     def single_scale_rpn_heads(self, res_dict, res_name_list):
@@ -295,8 +296,6 @@ class RCNN(object):
         return rois
 
     def generate_labels(self, input_rois):
-        #input_rois.persistable=True
-        #fluid.layers.Print(input_rois)
         outs = fluid.layers.generate_proposal_labels(
             rpn_rois=input_rois,
             gt_classes=self.gt_label,
@@ -317,7 +316,7 @@ class RCNN(object):
         self.bbox_targets = outs[2]
         self.bbox_inside_weights = outs[3]
         self.bbox_outside_weights = outs[4]
-
+        rois.persistable = True
         if cfg.MASK_ON:
             mask_out = fluid.layers.generate_mask_labels(
                 im_info=self.im_info,
@@ -352,6 +351,7 @@ class RCNN(object):
                                              name='cls_score_b',
                                              learning_rate=2.,
                                              regularizer=L2Decay(0.)))
+        self.cls_score.persistable = True
         self.bbox_pred = fluid.layers.fc(input=rcnn_out,
                                          size=4 * cfg.class_num,
                                          act=None,
@@ -364,6 +364,7 @@ class RCNN(object):
                                              name='bbox_pred_b',
                                              learning_rate=2.,
                                              regularizer=L2Decay(0.)))
+        self.bbox_pred.persistable = True
 
     def SuffixNet(self, conv5):
         mask_out = fluid.layers.conv2d_transpose(
@@ -456,58 +457,124 @@ class RCNN(object):
         loss_bbox = fluid.layers.reduce_mean(loss_bbox)
         return loss_cls, loss_bbox
 
-    def single_scale_rpn_loss(self):
+    def transform_rpn_input(self, rpn_cls_score, rpn_bbox_pred, anchor, var):
         rpn_cls_score_reshape = fluid.layers.transpose(
-            self.rpn_cls_score, perm=[0, 2, 3, 1])
+            rpn_cls_score, perm=[0, 2, 3, 1])
         rpn_bbox_pred_reshape = fluid.layers.transpose(
-            self.rpn_bbox_pred, perm=[0, 2, 3, 1])
+            rpn_bbox_pred, perm=[0, 2, 3, 1])
 
-        anchor_reshape = fluid.layers.reshape(self.anchor, shape=(-1, 4))
-        var_reshape = fluid.layers.reshape(self.var, shape=(-1, 4))
+        anchor_reshape = fluid.layers.reshape(anchor, shape=(-1, 4))
+        var_reshape = fluid.layers.reshape(var, shape=(-1, 4))
 
         rpn_cls_score_reshape = fluid.layers.reshape(
             x=rpn_cls_score_reshape, shape=(0, -1, 1))
         rpn_bbox_pred_reshape = fluid.layers.reshape(
             x=rpn_bbox_pred_reshape, shape=(0, -1, 4))
-        score_pred, loc_pred, score_tgt, loc_tgt, bbox_weight = \
-            fluid.layers.rpn_target_assign(
-                bbox_pred=rpn_bbox_pred_reshape,
-                cls_logits=rpn_cls_score_reshape,
-                anchor_box=anchor_reshape,
-                anchor_var=var_reshape,
-                gt_boxes=self.gt_box,
-                is_crowd=self.is_crowd,
-                im_info=self.im_info,
-                rpn_batch_size_per_im=cfg.TRAIN.rpn_batch_size_per_im,
-                rpn_straddle_thresh=cfg.TRAIN.rpn_straddle_thresh,
-                rpn_fg_fraction=cfg.TRAIN.rpn_fg_fraction,
-                rpn_positive_overlap=cfg.TRAIN.rpn_positive_overlap,
-                rpn_negative_overlap=cfg.TRAIN.rpn_negative_overlap,
-                use_random=self.use_random)
+
+        return rpn_cls_score_reshape, rpn_bbox_pred_reshape, anchor_reshape, var_reshape
+
+    def fpn_rpn_input(self):
+        rpn_cls_reshape_list = []
+        rpn_bbox_reshape_list = []
+        anchors_reshape_list = []
+        var_reshape_list = []
+        for i in range(len(self.rpn_fpn_list)):
+            single_rpn_input = self.transform_rpn_input(
+                self.rpn_fpn_list[i][0], self.rpn_fpn_list[i][1],
+                self.anchors_list[i], self.var_list[i])
+            rpn_cls_reshape_list.append(single_rpn_input[0])
+            rpn_bbox_reshape_list.append(single_rpn_input[1])
+            anchors_reshape_list.append(single_rpn_input[2])
+            var_reshape_list.append(single_rpn_input[3])
+
+        rpn_cls_input = fluid.layers.concat(rpn_cls_reshape_list, axis=1)
+        rpn_bbox_input = fluid.layers.concat(rpn_bbox_reshape_list, axis=1)
+        anchors_input = fluid.layers.concat(anchors_reshape_list)
+        var_input = fluid.layers.concat(var_reshape_list)
+        return rpn_cls_input, rpn_bbox_input, anchors_input, var_input
+
+    def get_rpn_loss(self,
+                     score_pred,
+                     loc_pred,
+                     score_tgt,
+                     loc_tgt,
+                     bbox_weight,
+                     level_score_weight=None,
+                     level_bbox_weight=None,
+                     lvl=None):
         score_tgt = fluid.layers.cast(x=score_tgt, dtype='float32')
+        score_tgt.stop_gradient = True
         rpn_cls_loss = fluid.layers.sigmoid_cross_entropy_with_logits(
             x=score_pred, label=score_tgt)
-        if cfg.FPN_ON:
+
+        if level_score_weight is not None:
+            level_score_weight = fluid.layers.cast(
+                x=level_score_weight, dtype='float32')
+            level_score_weight.stop_gradient = True
+            rpn_cls_loss = rpn_cls_loss * level_score_weight
+
         rpn_cls_loss = fluid.layers.reduce_sum(rpn_cls_loss)
         rpn_cls_loss = rpn_cls_loss / (cfg.TRAIN.im_per_batch *
                                        cfg.TRAIN.rpn_batch_size_per_im)
-        else:
-            rpn_cls_loss = fluid.layers.reduce_mean(
-                rpn_cls_loss, name='loss_rpn_cls')
-        rpn_reg_loss = fluid.layers.smooth_l1(
+        rpn_cls_loss.persistable = True
+
+        loc_tgt = fluid.layers.cast(x=loc_tgt, dtype='float32')
+        loc_tgt.stop_gradient = True
+        rpn_bbox_loss = fluid.layers.smooth_l1(
             x=loc_pred,
             y=loc_tgt,
             sigma=3.0,
             inside_weight=bbox_weight,
             outside_weight=bbox_weight)
-        rpn_reg_loss = fluid.layers.reduce_sum(
-            rpn_reg_loss, name='loss_rpn_bbox')
+
+        if level_bbox_weight is not None:
+            level_bbox_weight = fluid.layers.cast(
+                x=level_bbox_weight, dtype='float32')
+            level_bbox_weight.stop_gradient = True
+            rpn_bbox_loss = rpn_bbox_loss * level_bbox_weight
+
+        rpn_bbox_loss = fluid.layers.reduce_sum(rpn_bbox_loss)
         score_shape = fluid.layers.shape(score_tgt)
         score_shape = fluid.layers.cast(x=score_shape, dtype='float32')
         norm = fluid.layers.reduce_prod(score_shape)
         norm.stop_gradient = True
-        rpn_reg_loss = rpn_reg_loss / norm
-        return rpn_cls_loss, rpn_reg_loss
+
+        rpn_bbox_loss = rpn_bbox_loss / norm
+        return rpn_cls_loss, rpn_bbox_loss
+
+    def single_scale_rpn_loss(self):
+        rpn_input = self.transform_rpn_input(
+            self.rpn_cls_score, self.rpn_bbox_pred, self.anchor, self.var)
+        rpn_cls_score_reshape = rpn_input[0]
+        rpn_bbox_pred_reshape = rpn_input[1]
+        anchor_reshape = rpn_input[2]
+        var_reshape = rpn_input[3]
+        score_index, loc_index, score_tgt, loc_tgt, bbox_weight = \
+            fluid.layers.rpn_target_assign(
+                bbox_pred=rpn_bbox_pred_reshape,
+                cls_logits=rpn_cls_score_reshape,
+                anchor_box=anchor_reshape,
+                anchor_var=var_reshape,
+                gt_boxes=self.gt_box,
+                is_crowd=self.is_crowd,
+                im_info=self.im_info,
+                rpn_batch_size_per_im=cfg.TRAIN.rpn_batch_size_per_im,
+                rpn_straddle_thresh=cfg.TRAIN.rpn_straddle_thresh,
+                rpn_fg_fraction=cfg.TRAIN.rpn_fg_fraction,
+                rpn_positive_overlap=cfg.TRAIN.rpn_positive_overlap,
+                rpn_negative_overlap=cfg.TRAIN.rpn_negative_overlap,
+                use_random=self.use_random)
+        rpn_cls_score_reshape = fluid.layers.reshape(
+            x=rpn_cls_score_reshape, shape=(-1, 1))
+        rpn_bbox_pred_reshape = fluid.layers.reshape(
+            x=rpn_bbox_pred_reshape, shape=(-1, 4))
+        score_pred = fluid.layers.gather(rpn_cls_score_reshape, score_index)
+        loc_pred = fluid.layers.gather(rpn_bbox_pred_reshape, loc_index)
+
+        rpn_cls_loss, rpn_bbox_loss = self.get_rpn_loss(
+            score_pred, loc_pred, score_tgt, loc_tgt, bbox_weight)
+
+        return rpn_cls_loss, rpn_bbox_loss
 
     def fpn_rpn_loss(self):
         k_max = cfg.FPN_rpn_max_level
@@ -516,13 +583,63 @@ class RCNN(object):
         loss_rpn_bbox_fpn_name = []
         loss_rpn_cls_fpn_list = []
         loss_rpn_bbox_fpn_list = []
+        rpn_input = self.fpn_rpn_input()
+        rpn_cls_score_reshape = rpn_input[0]
+        rpn_bbox_pred_reshape = rpn_input[1]
+        anchor_reshape = rpn_input[2]
+        var_reshape = rpn_input[3]
+        score_index, loc_index, score_tgt, loc_tgt, bbox_weight = \
+            fluid.layers.rpn_target_assign(
+                bbox_pred=rpn_bbox_pred_reshape,
+                cls_logits=rpn_cls_score_reshape,
+                anchor_box=anchor_reshape,
+                anchor_var=var_reshape,
+                gt_boxes=self.gt_box,
+                is_crowd=self.is_crowd,
+                im_info=self.im_info,
+                rpn_batch_size_per_im=cfg.TRAIN.rpn_batch_size_per_im,
+                rpn_straddle_thresh=cfg.TRAIN.rpn_straddle_thresh,
+                rpn_fg_fraction=cfg.TRAIN.rpn_fg_fraction,
+                rpn_positive_overlap=cfg.TRAIN.rpn_positive_overlap,
+                rpn_negative_overlap=cfg.TRAIN.rpn_negative_overlap,
+                use_random=self.use_random)
+        rpn_cls_score_reshape = fluid.layers.reshape(
+            x=rpn_cls_score_reshape, shape=(-1, 1))
+        rpn_bbox_pred_reshape = fluid.layers.reshape(
+            x=rpn_bbox_pred_reshape, shape=(-1, 4))
+        score_pred = fluid.layers.gather(rpn_cls_score_reshape, score_index)
+        loc_pred = fluid.layers.gather(rpn_bbox_pred_reshape, loc_index)
+
         for lvl in range(k_min, k_max + 1):
             slvl = str(lvl)
-            self.rpn_cls_score = self.rpn_fpn_list[lvl - k_min][0]
-            self.rpn_bbox_pred = self.rpn_fpn_list[lvl - k_min][1]
-            self.anchor = self.anchors_list[lvl - k_min]
-            self.var = self.var_list[lvl - k_min]
-            loss_rpn_cls_fpn, loss_rpn_bbox_fpn = self.single_scale_rpn_loss()
+            if lvl == k_min:
+                anchor_num = self.anchor_num_list[lvl - k_min]
+                level_score_weight = fluid.layers.less_than(
+                    x=score_index, y=anchor_num)
+                level_loc_weight = fluid.layers.less_than(
+                    x=loc_index, y=anchor_num)
+            else:
+                anchor_num = self.anchor_num_list[lvl - k_min]
+                pre_anchor_num = self.anchor_num_list[lvl - k_min - 1]
+                level_score_weight_0 = fluid.layers.less_than(
+                    x=score_index, y=pre_anchor_num)
+                level_score_weight_1 = fluid.layers.less_than(
+                    x=score_index, y=anchor_num)
+                level_score_weight = fluid.layers.logical_xor(
+                    level_score_weight_0, level_score_weight_1)
+
+                level_loc_weight_0 = fluid.layers.less_than(
+                    x=loc_index, y=pre_anchor_num)
+                level_loc_weight_1 = fluid.layers.less_than(
+                    x=loc_index, y=anchor_num)
+                level_loc_weight = fluid.layers.logical_xor(level_loc_weight_0,
+                                                            level_loc_weight_1)
+
+            loss_rpn_cls_fpn, loss_rpn_bbox_fpn = self.get_rpn_loss(
+                score_pred, loc_pred, score_tgt, loc_tgt, bbox_weight,
+                level_score_weight, level_loc_weight, lvl)
+            loss_rpn_cls_fpn.persistable = True
+            loss_rpn_bbox_fpn.persistable = True
             loss_rpn_cls_fpn_name.append('loss_rpn_cls_fpn' + slvl)
             loss_rpn_bbox_fpn_name.append('loss_rpn_bbox_fpn' + slvl)
             loss_rpn_cls_fpn_list.append(loss_rpn_cls_fpn)

fluid/PaddleCV/rcnn/reader.py

@@ -111,6 +111,7 @@ def coco(mode,
                 roidb = roidb_perm[0]
                 roidb_cur += 1
                 roidb_perm.rotate(-1)
+                if '0000139' not in roidb['image']: continue
                 if roidb_cur >= len(roidbs):
                     if shuffle:
                         roidb_perm = deque(np.random.permutation(roidbs))