# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import absolute_import from __future__ import division from __future__ import print_function import paddle import paddle.nn as nn import paddle.nn.functional as F from ppdet.core.workspace import register from ppdet.modeling import ops from functools import partial __all__ = ['FCOSLoss', 'FCOSLossMILC', 'FCOSLossCR'] def flatten_tensor(inputs, channel_first=False): """ Flatten a Tensor Args: inputs (Tensor): 4-D Tensor with shape [N, C, H, W] or [N, H, W, C] channel_first (bool): If true the dimension order of Tensor is [N, C, H, W], otherwise is [N, H, W, C] Return: output_channel_last (Tensor): The flattened Tensor in channel_last style """ if channel_first: input_channel_last = paddle.transpose(inputs, perm=[0, 2, 3, 1]) else: input_channel_last = inputs output_channel_last = paddle.flatten( input_channel_last, start_axis=0, stop_axis=2) return output_channel_last @register class FCOSLoss(nn.Layer): """ FCOSLoss Args: loss_alpha (float): alpha in focal loss loss_gamma (float): gamma in focal loss iou_loss_type (str): location loss type, IoU/GIoU/LINEAR_IoU reg_weights (float): weight for location loss quality (str): quality branch, centerness/iou """ def __init__(self, loss_alpha=0.25, loss_gamma=2.0, iou_loss_type="giou", reg_weights=1.0, quality='centerness'): super(FCOSLoss, self).__init__() self.loss_alpha = loss_alpha self.loss_gamma = loss_gamma self.iou_loss_type = iou_loss_type self.reg_weights = reg_weights self.quality = quality def _iou_loss(self, pred, targets, positive_mask, weights=None, return_iou=False): """ Calculate the loss for location prediction Args: pred (Tensor): bounding boxes prediction targets (Tensor): targets for positive samples positive_mask (Tensor): mask of positive samples weights (Tensor): weights for each positive samples Return: loss (Tensor): location loss """ plw = pred[:, 0] * positive_mask pth = pred[:, 1] * positive_mask prw = pred[:, 2] * positive_mask pbh = pred[:, 3] * positive_mask tlw = targets[:, 0] * positive_mask tth = targets[:, 1] * positive_mask trw = targets[:, 2] * positive_mask tbh = targets[:, 3] * positive_mask tlw.stop_gradient = True trw.stop_gradient = True tth.stop_gradient = True tbh.stop_gradient = True ilw = paddle.minimum(plw, tlw) irw = paddle.minimum(prw, trw) ith = paddle.minimum(pth, tth) ibh = paddle.minimum(pbh, tbh) clw = paddle.maximum(plw, tlw) crw = paddle.maximum(prw, trw) cth = paddle.maximum(pth, tth) cbh = paddle.maximum(pbh, tbh) area_predict = (plw + prw) * (pth + pbh) area_target = (tlw + trw) * (tth + tbh) area_inter = (ilw + irw) * (ith + ibh) ious = (area_inter + 1.0) / ( area_predict + area_target - area_inter + 1.0) ious = ious * positive_mask if return_iou: return ious if self.iou_loss_type.lower() == "linear_iou": loss = 1.0 - ious elif self.iou_loss_type.lower() == "giou": area_uniou = area_predict + area_target - area_inter area_circum = (clw + crw) * (cth + cbh) + 1e-7 giou = ious - (area_circum - area_uniou) / area_circum loss = 1.0 - giou elif self.iou_loss_type.lower() == "iou": loss = 0.0 - paddle.log(ious) else: raise KeyError if weights is not None: loss = loss * weights return loss def forward(self, cls_logits, bboxes_reg, centerness, tag_labels, tag_bboxes, tag_center): """ Calculate the loss for classification, location and centerness Args: cls_logits (list): list of Tensor, which is predicted score for all anchor points with shape [N, M, C] bboxes_reg (list): list of Tensor, which is predicted offsets for all anchor points with shape [N, M, 4] centerness (list): list of Tensor, which is predicted centerness for all anchor points with shape [N, M, 1] tag_labels (list): list of Tensor, which is category targets for each anchor point tag_bboxes (list): list of Tensor, which is bounding boxes targets for positive samples tag_center (list): list of Tensor, which is centerness targets for positive samples Return: loss (dict): loss composed by classification loss, bounding box """ cls_logits_flatten_list = [] bboxes_reg_flatten_list = [] centerness_flatten_list = [] tag_labels_flatten_list = [] tag_bboxes_flatten_list = [] tag_center_flatten_list = [] num_lvl = len(cls_logits) for lvl in range(num_lvl): cls_logits_flatten_list.append( flatten_tensor(cls_logits[lvl], True)) bboxes_reg_flatten_list.append( flatten_tensor(bboxes_reg[lvl], True)) centerness_flatten_list.append( flatten_tensor(centerness[lvl], True)) tag_labels_flatten_list.append( flatten_tensor(tag_labels[lvl], False)) tag_bboxes_flatten_list.append( flatten_tensor(tag_bboxes[lvl], False)) tag_center_flatten_list.append( flatten_tensor(tag_center[lvl], False)) cls_logits_flatten = paddle.concat(cls_logits_flatten_list, axis=0) bboxes_reg_flatten = paddle.concat(bboxes_reg_flatten_list, axis=0) centerness_flatten = paddle.concat(centerness_flatten_list, axis=0) tag_labels_flatten = paddle.concat(tag_labels_flatten_list, axis=0) tag_bboxes_flatten = paddle.concat(tag_bboxes_flatten_list, axis=0) tag_center_flatten = paddle.concat(tag_center_flatten_list, axis=0) tag_labels_flatten.stop_gradient = True tag_bboxes_flatten.stop_gradient = True tag_center_flatten.stop_gradient = True mask_positive_bool = tag_labels_flatten > 0 mask_positive_bool.stop_gradient = True mask_positive_float = paddle.cast(mask_positive_bool, dtype="float32") mask_positive_float.stop_gradient = True num_positive_fp32 = paddle.sum(mask_positive_float) num_positive_fp32.stop_gradient = True num_positive_int32 = paddle.cast(num_positive_fp32, dtype="int32") num_positive_int32 = num_positive_int32 * 0 + 1 num_positive_int32.stop_gradient = True normalize_sum = paddle.sum(tag_center_flatten * mask_positive_float) normalize_sum.stop_gradient = True # 1. cls_logits: sigmoid_focal_loss # expand onehot labels num_classes = cls_logits_flatten.shape[-1] tag_labels_flatten = paddle.squeeze(tag_labels_flatten, axis=-1) tag_labels_flatten_bin = F.one_hot( tag_labels_flatten, num_classes=1 + num_classes) tag_labels_flatten_bin = tag_labels_flatten_bin[:, 1:] # sigmoid_focal_loss cls_loss = F.sigmoid_focal_loss( cls_logits_flatten, tag_labels_flatten_bin) / num_positive_fp32 if self.quality == 'centerness': # 2. bboxes_reg: giou_loss mask_positive_float = paddle.squeeze(mask_positive_float, axis=-1) tag_center_flatten = paddle.squeeze(tag_center_flatten, axis=-1) reg_loss = self._iou_loss( bboxes_reg_flatten, tag_bboxes_flatten, mask_positive_float, weights=tag_center_flatten) reg_loss = reg_loss * mask_positive_float / normalize_sum # 3. centerness: sigmoid_cross_entropy_with_logits_loss centerness_flatten = paddle.squeeze(centerness_flatten, axis=-1) quality_loss = ops.sigmoid_cross_entropy_with_logits( centerness_flatten, tag_center_flatten) quality_loss = quality_loss * mask_positive_float / num_positive_fp32 elif self.quality == 'iou': # 2. bboxes_reg: giou_loss mask_positive_float = paddle.squeeze(mask_positive_float, axis=-1) tag_center_flatten = paddle.squeeze(tag_center_flatten, axis=-1) reg_loss = self._iou_loss( bboxes_reg_flatten, tag_bboxes_flatten, mask_positive_float, weights=None) reg_loss = reg_loss * mask_positive_float / num_positive_fp32 # num_positive_fp32 is num_foreground # 3. centerness: sigmoid_cross_entropy_with_logits_loss centerness_flatten = paddle.squeeze(centerness_flatten, axis=-1) gt_ious = self._iou_loss( bboxes_reg_flatten, tag_bboxes_flatten, mask_positive_float, weights=None, return_iou=True) quality_loss = ops.sigmoid_cross_entropy_with_logits( centerness_flatten, gt_ious) quality_loss = quality_loss * mask_positive_float / num_positive_fp32 else: raise Exception(f'Unknown quality type: {self.quality}') loss_all = { "loss_cls": paddle.sum(cls_loss), "loss_box": paddle.sum(reg_loss), "loss_quality": paddle.sum(quality_loss), } return loss_all @register class FCOSLossMILC(FCOSLoss): """ FCOSLossMILC for ARSL in semi-det(ssod) Args: loss_alpha (float): alpha in focal loss loss_gamma (float): gamma in focal loss iou_loss_type (str): location loss type, IoU/GIoU/LINEAR_IoU reg_weights (float): weight for location loss """ def __init__(self, loss_alpha=0.25, loss_gamma=2.0, iou_loss_type="giou", reg_weights=1.0): super(FCOSLossMILC, self).__init__() self.loss_alpha = loss_alpha self.loss_gamma = loss_gamma self.iou_loss_type = iou_loss_type self.reg_weights = reg_weights def iou_loss(self, pred, targets, weights=None, avg_factor=None): """ Calculate the loss for location prediction Args: pred (Tensor): bounding boxes prediction targets (Tensor): targets for positive samples weights (Tensor): weights for each positive samples Return: loss (Tensor): location loss """ plw = pred[:, 0] pth = pred[:, 1] prw = pred[:, 2] pbh = pred[:, 3] tlw = targets[:, 0] tth = targets[:, 1] trw = targets[:, 2] tbh = targets[:, 3] tlw.stop_gradient = True trw.stop_gradient = True tth.stop_gradient = True tbh.stop_gradient = True ilw = paddle.minimum(plw, tlw) irw = paddle.minimum(prw, trw) ith = paddle.minimum(pth, tth) ibh = paddle.minimum(pbh, tbh) clw = paddle.maximum(plw, tlw) crw = paddle.maximum(prw, trw) cth = paddle.maximum(pth, tth) cbh = paddle.maximum(pbh, tbh) area_predict = (plw + prw) * (pth + pbh) area_target = (tlw + trw) * (tth + tbh) area_inter = (ilw + irw) * (ith + ibh) ious = (area_inter + 1.0) / ( area_predict + area_target - area_inter + 1.0) ious = ious if self.iou_loss_type.lower() == "linear_iou": loss = 1.0 - ious elif self.iou_loss_type.lower() == "giou": area_uniou = area_predict + area_target - area_inter area_circum = (clw + crw) * (cth + cbh) + 1e-7 giou = ious - (area_circum - area_uniou) / area_circum loss = 1.0 - giou elif self.iou_loss_type.lower() == "iou": loss = 0.0 - paddle.log(ious) else: raise KeyError if weights is not None: loss = loss * weights loss = paddle.sum(loss) if avg_factor is not None: loss = loss / avg_factor return loss # temp function: calcualate iou between bbox and target def _bbox_overlap_align(self, pred, targets): assert pred.shape[0] == targets.shape[0], \ 'the pred should be aligned with target.' plw = pred[:, 0] pth = pred[:, 1] prw = pred[:, 2] pbh = pred[:, 3] tlw = targets[:, 0] tth = targets[:, 1] trw = targets[:, 2] tbh = targets[:, 3] ilw = paddle.minimum(plw, tlw) irw = paddle.minimum(prw, trw) ith = paddle.minimum(pth, tth) ibh = paddle.minimum(pbh, tbh) area_predict = (plw + prw) * (pth + pbh) area_target = (tlw + trw) * (tth + tbh) area_inter = (ilw + irw) * (ith + ibh) ious = (area_inter + 1.0) / ( area_predict + area_target - area_inter + 1.0) return ious def iou_based_soft_label_loss(self, pred, target, alpha=0.75, gamma=2.0, iou_weighted=False, implicit_iou=None, avg_factor=None): assert pred.shape == target.shape pred = F.sigmoid(pred) target = target.cast(pred.dtype) if implicit_iou is not None: pred = pred * implicit_iou if iou_weighted: focal_weight = (pred - target).abs().pow(gamma) * target * (target > 0.0).cast('float32') + \ alpha * (pred - target).abs().pow(gamma) * \ (target <= 0.0).cast('float32') else: focal_weight = (pred - target).abs().pow(gamma) * (target > 0.0).cast('float32') + \ alpha * (pred - target).abs().pow(gamma) * \ (target <= 0.0).cast('float32') # focal loss loss = F.binary_cross_entropy( pred, target, reduction='none') * focal_weight if avg_factor is not None: loss = loss / avg_factor return loss def forward(self, cls_logits, bboxes_reg, centerness, tag_labels, tag_bboxes, tag_center): """ Calculate the loss for classification, location and centerness Args: cls_logits (list): list of Tensor, which is predicted score for all anchor points with shape [N, M, C] bboxes_reg (list): list of Tensor, which is predicted offsets for all anchor points with shape [N, M, 4] centerness (list): list of Tensor, which is predicted centerness for all anchor points with shape [N, M, 1] tag_labels (list): list of Tensor, which is category targets for each anchor point tag_bboxes (list): list of Tensor, which is bounding boxes targets for positive samples tag_center (list): list of Tensor, which is centerness targets for positive samples Return: loss (dict): loss composed by classification loss, bounding box """ cls_logits_flatten_list = [] bboxes_reg_flatten_list = [] centerness_flatten_list = [] tag_labels_flatten_list = [] tag_bboxes_flatten_list = [] tag_center_flatten_list = [] num_lvl = len(cls_logits) for lvl in range(num_lvl): cls_logits_flatten_list.append( flatten_tensor(cls_logits[lvl], True)) bboxes_reg_flatten_list.append( flatten_tensor(bboxes_reg[lvl], True)) centerness_flatten_list.append( flatten_tensor(centerness[lvl], True)) tag_labels_flatten_list.append( flatten_tensor(tag_labels[lvl], False)) tag_bboxes_flatten_list.append( flatten_tensor(tag_bboxes[lvl], False)) tag_center_flatten_list.append( flatten_tensor(tag_center[lvl], False)) cls_logits_flatten = paddle.concat(cls_logits_flatten_list, axis=0) bboxes_reg_flatten = paddle.concat(bboxes_reg_flatten_list, axis=0) centerness_flatten = paddle.concat(centerness_flatten_list, axis=0) tag_labels_flatten = paddle.concat(tag_labels_flatten_list, axis=0) tag_bboxes_flatten = paddle.concat(tag_bboxes_flatten_list, axis=0) tag_center_flatten = paddle.concat(tag_center_flatten_list, axis=0) tag_labels_flatten.stop_gradient = True tag_bboxes_flatten.stop_gradient = True tag_center_flatten.stop_gradient = True # find positive index mask_positive_bool = tag_labels_flatten > 0 mask_positive_bool.stop_gradient = True mask_positive_float = paddle.cast(mask_positive_bool, dtype="float32") mask_positive_float.stop_gradient = True num_positive_fp32 = paddle.sum(mask_positive_float) num_positive_fp32.stop_gradient = True num_positive_int32 = paddle.cast(num_positive_fp32, dtype="int32") num_positive_int32 = num_positive_int32 * 0 + 1 num_positive_int32.stop_gradient = True # centerness target is used as reg weight normalize_sum = paddle.sum(tag_center_flatten * mask_positive_float) normalize_sum.stop_gradient = True # 1. IoU-Based soft label loss # calculate iou with paddle.no_grad(): pos_ind = paddle.nonzero( tag_labels_flatten.reshape([-1]) > 0).reshape([-1]) pos_pred = bboxes_reg_flatten[pos_ind] pos_target = tag_bboxes_flatten[pos_ind] bbox_iou = self._bbox_overlap_align(pos_pred, pos_target) # pos labels pos_labels = tag_labels_flatten[pos_ind].squeeze(1) cls_target = paddle.zeros(cls_logits_flatten.shape) cls_target[pos_ind, pos_labels - 1] = bbox_iou cls_loss = self.iou_based_soft_label_loss( cls_logits_flatten, cls_target, implicit_iou=F.sigmoid(centerness_flatten), avg_factor=num_positive_fp32) # 2. bboxes_reg: giou_loss mask_positive_float = paddle.squeeze(mask_positive_float, axis=-1) tag_center_flatten = paddle.squeeze(tag_center_flatten, axis=-1) reg_loss = self._iou_loss( bboxes_reg_flatten, tag_bboxes_flatten, mask_positive_float, weights=tag_center_flatten) reg_loss = reg_loss * mask_positive_float / normalize_sum # 3. iou loss pos_iou_pred = paddle.squeeze(centerness_flatten, axis=-1)[pos_ind] loss_iou = ops.sigmoid_cross_entropy_with_logits(pos_iou_pred, bbox_iou) loss_iou = loss_iou / num_positive_fp32 * 0.5 loss_all = { "loss_cls": paddle.sum(cls_loss), "loss_box": paddle.sum(reg_loss), 'loss_iou': paddle.sum(loss_iou), } return loss_all # Concat multi-level feature maps by image def levels_to_images(mlvl_tensor): batch_size = mlvl_tensor[0].shape[0] batch_list = [[] for _ in range(batch_size)] channels = mlvl_tensor[0].shape[1] for t in mlvl_tensor: t = t.transpose([0, 2, 3, 1]) t = t.reshape([batch_size, -1, channels]) for img in range(batch_size): batch_list[img].append(t[img]) return [paddle.concat(item, axis=0) for item in batch_list] def multi_apply(func, *args, **kwargs): """Apply function to a list of arguments. Note: This function applies the ``func`` to multiple inputs and map the multiple outputs of the ``func`` into different list. Each list contains the same type of outputs corresponding to different inputs. Args: func (Function): A function that will be applied to a list of arguments Returns: tuple(list): A tuple containing multiple list, each list contains \ a kind of returned results by the function """ pfunc = partial(func, **kwargs) if kwargs else func map_results = map(pfunc, *args) return tuple(map(list, zip(*map_results))) @register class FCOSLossCR(FCOSLossMILC): """ FCOSLoss of Consistency Regularization """ def __init__(self, iou_loss_type="giou", cls_weight=2.0, reg_weight=2.0, iou_weight=0.5, hard_neg_mining_flag=True): super(FCOSLossCR, self).__init__() self.iou_loss_type = iou_loss_type self.cls_weight = cls_weight self.reg_weight = reg_weight self.iou_weight = iou_weight self.hard_neg_mining_flag = hard_neg_mining_flag def iou_loss(self, pred, targets, weights=None, avg_factor=None): """ Calculate the loss for location prediction Args: pred (Tensor): bounding boxes prediction targets (Tensor): targets for positive samples weights (Tensor): weights for each positive samples Return: loss (Tensor): location loss """ plw = pred[:, 0] pth = pred[:, 1] prw = pred[:, 2] pbh = pred[:, 3] tlw = targets[:, 0] tth = targets[:, 1] trw = targets[:, 2] tbh = targets[:, 3] tlw.stop_gradient = True trw.stop_gradient = True tth.stop_gradient = True tbh.stop_gradient = True ilw = paddle.minimum(plw, tlw) irw = paddle.minimum(prw, trw) ith = paddle.minimum(pth, tth) ibh = paddle.minimum(pbh, tbh) clw = paddle.maximum(plw, tlw) crw = paddle.maximum(prw, trw) cth = paddle.maximum(pth, tth) cbh = paddle.maximum(pbh, tbh) area_predict = (plw + prw) * (pth + pbh) area_target = (tlw + trw) * (tth + tbh) area_inter = (ilw + irw) * (ith + ibh) ious = (area_inter + 1.0) / ( area_predict + area_target - area_inter + 1.0) ious = ious if self.iou_loss_type.lower() == "linear_iou": loss = 1.0 - ious elif self.iou_loss_type.lower() == "giou": area_uniou = area_predict + area_target - area_inter area_circum = (clw + crw) * (cth + cbh) + 1e-7 giou = ious - (area_circum - area_uniou) / area_circum loss = 1.0 - giou elif self.iou_loss_type.lower() == "iou": loss = 0.0 - paddle.log(ious) else: raise KeyError if weights is not None: loss = loss * weights loss = paddle.sum(loss) if avg_factor is not None: loss = loss / avg_factor return loss # calcualate iou between bbox and target def bbox_overlap_align(self, pred, targets): assert pred.shape[0] == targets.shape[0], \ 'the pred should be aligned with target.' plw = pred[:, 0] pth = pred[:, 1] prw = pred[:, 2] pbh = pred[:, 3] tlw = targets[:, 0] tth = targets[:, 1] trw = targets[:, 2] tbh = targets[:, 3] ilw = paddle.minimum(plw, tlw) irw = paddle.minimum(prw, trw) ith = paddle.minimum(pth, tth) ibh = paddle.minimum(pbh, tbh) area_predict = (plw + prw) * (pth + pbh) area_target = (tlw + trw) * (tth + tbh) area_inter = (ilw + irw) * (ith + ibh) ious = (area_inter + 1.0) / ( area_predict + area_target - area_inter + 1.0) return ious # cls loss: iou-based soft lable with joint iou def quality_focal_loss(self, stu_cls, targets, quality=None, weights=None, alpha=0.75, gamma=2.0, avg_factor='sum'): stu_cls = F.sigmoid(stu_cls) if quality is not None: stu_cls = stu_cls * F.sigmoid(quality) focal_weight = (stu_cls - targets).abs().pow(gamma) * (targets > 0.0).cast('float32') + \ alpha * (stu_cls - targets).abs().pow(gamma) * \ (targets <= 0.0).cast('float32') loss = F.binary_cross_entropy( stu_cls, targets, reduction='none') * focal_weight if weights is not None: loss = loss * weights.reshape([-1, 1]) loss = paddle.sum(loss) if avg_factor is not None: loss = loss / avg_factor return loss # generate points according to feature maps def compute_locations_by_level(self, fpn_stride, h, w): """ Compute locations of anchor points of each FPN layer Return: Anchor points locations of current FPN feature map """ shift_x = paddle.arange(0, w * fpn_stride, fpn_stride) shift_y = paddle.arange(0, h * fpn_stride, fpn_stride) shift_x = paddle.unsqueeze(shift_x, axis=0) shift_y = paddle.unsqueeze(shift_y, axis=1) shift_x = paddle.expand(shift_x, shape=[h, w]) shift_y = paddle.expand(shift_y, shape=[h, w]) shift_x = paddle.reshape(shift_x, shape=[-1]) shift_y = paddle.reshape(shift_y, shape=[-1]) location = paddle.stack( [shift_x, shift_y], axis=-1) + float(fpn_stride) / 2 return location # decode bbox from ltrb to x1y1x2y2 def decode_bbox(self, ltrb, points): assert ltrb.shape[0] == points.shape[0], \ "When decoding bbox in one image, the num of loc should be same with points." bbox_decoding = paddle.stack( [ points[:, 0] - ltrb[:, 0], points[:, 1] - ltrb[:, 1], points[:, 0] + ltrb[:, 2], points[:, 1] + ltrb[:, 3] ], axis=1) return bbox_decoding # encode bbox from x1y1x2y2 to ltrb def encode_bbox(self, bbox, points): assert bbox.shape[0] == points.shape[0], \ "When encoding bbox in one image, the num of bbox should be same with points." bbox_encoding = paddle.stack( [ points[:, 0] - bbox[:, 0], points[:, 1] - bbox[:, 1], bbox[:, 2] - points[:, 0], bbox[:, 3] - points[:, 1] ], axis=1) return bbox_encoding def calcualate_iou(self, gt_bbox, predict_bbox): # bbox area gt_area = (gt_bbox[:, 2] - gt_bbox[:, 0]) * \ (gt_bbox[:, 3] - gt_bbox[:, 1]) predict_area = (predict_bbox[:, 2] - predict_bbox[:, 0]) * \ (predict_bbox[:, 3] - predict_bbox[:, 1]) # overlop area lt = paddle.fmax(gt_bbox[:, None, :2], predict_bbox[None, :, :2]) rb = paddle.fmin(gt_bbox[:, None, 2:], predict_bbox[None, :, 2:]) wh = paddle.clip(rb - lt, min=0) overlap = wh[..., 0] * wh[..., 1] # iou iou = overlap / (gt_area[:, None] + predict_area[None, :] - overlap) return iou # select potential positives from hard negatives def hard_neg_mining(self, cls_score, loc_ltrb, quality, pos_ind, hard_neg_ind, loc_mask, loc_targets, iou_thresh=0.6): # get points locations and strides points_list = [] strides_list = [] scale_list = [] scale = [0, 1, 2, 3, 4] for fpn_scale, fpn_stride, HW in zip(scale, self.fpn_stride, self.lvl_hw): h, w = HW lvl_points = self.compute_locations_by_level(fpn_stride, h, w) points_list.append(lvl_points) lvl_strides = paddle.full([h * w, 1], fpn_stride) strides_list.append(lvl_strides) lvl_scales = paddle.full([h * w, 1], fpn_scale) scale_list.append(lvl_scales) points = paddle.concat(points_list, axis=0) strides = paddle.concat(strides_list, axis=0) scales = paddle.concat(scale_list, axis=0) # cls scores cls_vals = F.sigmoid(cls_score) * F.sigmoid(quality) max_vals = paddle.max(cls_vals, axis=-1) class_ind = paddle.argmax(cls_vals, axis=-1) ### calculate iou between positive and hard negative # decode pos bbox pos_cls = max_vals[pos_ind] pos_loc = loc_ltrb[pos_ind].reshape([-1, 4]) pos_strides = strides[pos_ind] pos_points = points[pos_ind].reshape([-1, 2]) pos_loc = pos_loc * pos_strides pos_bbox = self.decode_bbox(pos_loc, pos_points) pos_scales = scales[pos_ind] # decode hard negative bbox hard_neg_loc = loc_ltrb[hard_neg_ind].reshape([-1, 4]) hard_neg_strides = strides[hard_neg_ind] hard_neg_points = points[hard_neg_ind].reshape([-1, 2]) hard_neg_loc = hard_neg_loc * hard_neg_strides hard_neg_bbox = self.decode_bbox(hard_neg_loc, hard_neg_points) hard_neg_scales = scales[hard_neg_ind] # iou between pos bbox and hard negative bbox hard_neg_pos_iou = self.calcualate_iou(hard_neg_bbox, pos_bbox) ### select potential positives from hard negatives # scale flag scale_temp = paddle.abs( pos_scales.reshape([-1])[None, :] - hard_neg_scales.reshape([-1]) [:, None]) scale_flag = (scale_temp <= 1.) # iou flag iou_flag = (hard_neg_pos_iou >= iou_thresh) # same class flag pos_class = class_ind[pos_ind] hard_neg_class = class_ind[hard_neg_ind] class_flag = pos_class[None, :] - hard_neg_class[:, None] class_flag = (class_flag == 0) # hard negative point inside positive bbox flag ltrb_temp = paddle.stack( [ hard_neg_points[:, None, 0] - pos_bbox[None, :, 0], hard_neg_points[:, None, 1] - pos_bbox[None, :, 1], pos_bbox[None, :, 2] - hard_neg_points[:, None, 0], pos_bbox[None, :, 3] - hard_neg_points[:, None, 1] ], axis=-1) inside_flag = ltrb_temp.min(axis=-1) > 0 # reset iou valid_flag = (iou_flag & class_flag & inside_flag & scale_flag) invalid_iou = paddle.zeros_like(hard_neg_pos_iou) hard_neg_pos_iou = paddle.where(valid_flag, hard_neg_pos_iou, invalid_iou) pos_hard_neg_max_iou = hard_neg_pos_iou.max(axis=-1) # selece potential pos potential_pos_ind = (pos_hard_neg_max_iou > 0.) num_potential_pos = paddle.nonzero(potential_pos_ind).shape[0] if num_potential_pos == 0: return None ### calculate loc target：aggregate all matching bboxes as the bbox targets of potential pos # prepare data potential_points = hard_neg_points[potential_pos_ind].reshape([-1, 2]) potential_strides = hard_neg_strides[potential_pos_ind] potential_valid_flag = valid_flag[potential_pos_ind] potential_pos_ind = hard_neg_ind[potential_pos_ind] # get cls and box of matching positives pos_cls = max_vals[pos_ind] expand_pos_bbox = paddle.expand( pos_bbox, shape=[num_potential_pos, pos_bbox.shape[0], pos_bbox.shape[1]]) expand_pos_cls = paddle.expand( pos_cls, shape=[num_potential_pos, pos_cls.shape[0]]) invalid_cls = paddle.zeros_like(expand_pos_cls) expand_pos_cls = paddle.where(potential_valid_flag, expand_pos_cls, invalid_cls) expand_pos_cls = paddle.unsqueeze(expand_pos_cls, axis=-1) # aggregate box based on cls_score agg_bbox = (expand_pos_bbox * expand_pos_cls).sum(axis=1) \ / expand_pos_cls.sum(axis=1) agg_ltrb = self.encode_bbox(agg_bbox, potential_points) agg_ltrb = agg_ltrb / potential_strides # loc target for all pos loc_targets[potential_pos_ind] = agg_ltrb loc_mask[potential_pos_ind] = 1. return loc_mask, loc_targets # get training targets def get_targets_per_img(self, tea_cls, tea_loc, tea_iou, stu_cls, stu_loc, stu_iou): ### sample selection # prepare datas tea_cls_scores = F.sigmoid(tea_cls) * F.sigmoid(tea_iou) class_ind = paddle.argmax(tea_cls_scores, axis=-1) max_vals = paddle.max(tea_cls_scores, axis=-1) cls_mask = paddle.zeros_like( max_vals ) # set cls valid mask: pos is 1, hard_negative and negative are 0. num_pos, num_hard_neg = 0, 0 # mean-std selection # using nonzero to turn index from bool to int, because the index will be used to compose two-dim index in following. # using squeeze rather than reshape to avoid errors when no score is larger than thresh. candidate_ind = paddle.nonzero(max_vals >= 0.1).squeeze(axis=-1) num_candidate = candidate_ind.shape[0] if num_candidate > 0: # pos thresh = mean + std to select pos samples candidate_score = max_vals[candidate_ind] candidate_score_mean = candidate_score.mean() candidate_score_std = candidate_score.std() pos_thresh = (candidate_score_mean + candidate_score_std).clip( max=0.4) # select pos pos_ind = paddle.nonzero(max_vals >= pos_thresh).squeeze(axis=-1) num_pos = pos_ind.shape[0] # select hard negatives as potential pos hard_neg_ind = (max_vals >= 0.1) & (max_vals < pos_thresh) hard_neg_ind = paddle.nonzero(hard_neg_ind).squeeze(axis=-1) num_hard_neg = hard_neg_ind.shape[0] # if not positive, directly select top-10 as pos. if (num_pos == 0): num_pos = 10 _, pos_ind = paddle.topk(max_vals, k=num_pos) cls_mask[pos_ind] = 1. ### Consistency Regularization Training targets # cls targets pos_class_ind = class_ind[pos_ind] cls_targets = paddle.zeros_like(tea_cls) cls_targets[pos_ind, pos_class_ind] = tea_cls_scores[pos_ind, pos_class_ind] # hard negative cls target if num_hard_neg != 0: cls_targets[hard_neg_ind] = tea_cls_scores[hard_neg_ind] # loc targets loc_targets = paddle.zeros_like(tea_loc) loc_targets[pos_ind] = tea_loc[pos_ind] # iou targets iou_targets = paddle.zeros( shape=[tea_iou.shape[0]], dtype=tea_iou.dtype) iou_targets[pos_ind] = F.sigmoid( paddle.squeeze( tea_iou, axis=-1)[pos_ind]) loc_mask = cls_mask.clone() # select potential positive from hard negatives for loc_task training if (num_hard_neg > 0) and self.hard_neg_mining_flag: results = self.hard_neg_mining(tea_cls, tea_loc, tea_iou, pos_ind, hard_neg_ind, loc_mask, loc_targets) if results is not None: loc_mask, loc_targets = results loc_pos_ind = paddle.nonzero(loc_mask > 0.).squeeze(axis=-1) iou_targets[loc_pos_ind] = F.sigmoid( paddle.squeeze( tea_iou, axis=-1)[loc_pos_ind]) return cls_mask, loc_mask, \ cls_targets, loc_targets, iou_targets def forward(self, student_prediction, teacher_prediction): stu_cls_lvl, stu_loc_lvl, stu_iou_lvl = student_prediction tea_cls_lvl, tea_loc_lvl, tea_iou_lvl, self.fpn_stride = teacher_prediction # H and W of level (used for aggregating targets) self.lvl_hw = [] for t in tea_cls_lvl: _, _, H, W = t.shape self.lvl_hw.append([H, W]) # levels to images stu_cls_img = levels_to_images(stu_cls_lvl) stu_loc_img = levels_to_images(stu_loc_lvl) stu_iou_img = levels_to_images(stu_iou_lvl) tea_cls_img = levels_to_images(tea_cls_lvl) tea_loc_img = levels_to_images(tea_loc_lvl) tea_iou_img = levels_to_images(tea_iou_lvl) with paddle.no_grad(): cls_mask, loc_mask, \ cls_targets, loc_targets, iou_targets = multi_apply( self.get_targets_per_img, tea_cls_img, tea_loc_img, tea_iou_img, stu_cls_img, stu_loc_img, stu_iou_img ) # flatten preditction stu_cls = paddle.concat(stu_cls_img, axis=0) stu_loc = paddle.concat(stu_loc_img, axis=0) stu_iou = paddle.concat(stu_iou_img, axis=0) # flatten targets cls_mask = paddle.concat(cls_mask, axis=0) loc_mask = paddle.concat(loc_mask, axis=0) cls_targets = paddle.concat(cls_targets, axis=0) loc_targets = paddle.concat(loc_targets, axis=0) iou_targets = paddle.concat(iou_targets, axis=0) ### Training Weights and avg factor # find positives cls_pos_ind = paddle.nonzero(cls_mask > 0.).squeeze(axis=-1) loc_pos_ind = paddle.nonzero(loc_mask > 0.).squeeze(axis=-1) # cls weight cls_sample_weights = paddle.ones([cls_targets.shape[0]]) cls_avg_factor = paddle.max(cls_targets[cls_pos_ind], axis=-1).sum().item() # loc weight loc_sample_weights = paddle.max(cls_targets[loc_pos_ind], axis=-1) loc_avg_factor = loc_sample_weights.sum().item() # iou weight iou_sample_weights = paddle.ones([loc_pos_ind.shape[0]]) iou_avg_factor = loc_pos_ind.shape[0] ### unsupervised loss # cls loss loss_cls = self.quality_focal_loss( stu_cls, cls_targets, quality=stu_iou, weights=cls_sample_weights, avg_factor=cls_avg_factor) * self.cls_weight # iou loss pos_stu_iou = paddle.squeeze(stu_iou, axis=-1)[loc_pos_ind] pos_iou_targets = iou_targets[loc_pos_ind] loss_iou = F.binary_cross_entropy( F.sigmoid(pos_stu_iou), pos_iou_targets, reduction='none') * iou_sample_weights loss_iou = loss_iou.sum() / iou_avg_factor * self.iou_weight # box loss pos_stu_loc = stu_loc[loc_pos_ind] pos_loc_targets = loc_targets[loc_pos_ind] loss_box = self.iou_loss( pos_stu_loc, pos_loc_targets, weights=loc_sample_weights, avg_factor=loc_avg_factor) loss_box = loss_box * self.reg_weight loss_all = { "loss_cls": loss_cls, "loss_box": loss_box, "loss_iou": loss_iou, } return loss_all