metabinloss.py

#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# reference: https://arxiv.org/abs/2011.14670

import copy
import numpy as np
import paddle
from paddle import nn
from paddle.nn import functional as F

from .dist_loss import cosine_similarity
from .celoss import CELoss


def euclidean_dist(x, y):
    m, n = x.shape[0], y.shape[0]
    xx = paddle.pow(x, 2).sum(1, keepdim=True).expand([m, n])
    yy = paddle.pow(y, 2).sum(1, keepdim=True).expand([n, m]).t()
    dist = xx + yy - 2 * paddle.matmul(x, y.t())
    dist = dist.clip(min=1e-12).sqrt()  # for numerical stability
    return dist


def hard_example_mining(dist_mat, is_pos, is_neg):
    """For each anchor, find the hardest positive and negative sample.
    Args:
      dist_mat: pairwise distance between samples, shape [N, M]
      is_pos: positive index with shape [N, M]
      is_neg: negative index with shape [N, M]
    Returns:
      dist_ap: distance(anchor, positive); shape [N, 1]
      dist_an: distance(anchor, negative); shape [N, 1]
    """

    inf = float("inf")

    def _masked_max(tensor, mask, axis):
        masked = paddle.multiply(tensor, mask.astype(tensor.dtype))
        neg_inf = paddle.zeros_like(tensor)
        neg_inf.stop_gradient = True
        neg_inf[paddle.logical_not(mask)] = -inf
        return paddle.max(masked + neg_inf, axis=axis, keepdim=True)

    def _masked_min(tensor, mask, axis):
        masked = paddle.multiply(tensor, mask.astype(tensor.dtype))
        pos_inf = paddle.zeros_like(tensor)
        pos_inf.stop_gradient = True
        pos_inf[paddle.logical_not(mask)] = inf
        return paddle.min(masked + pos_inf, axis=axis, keepdim=True)

    assert len(dist_mat.shape) == 2
    dist_ap = _masked_max(dist_mat, is_pos, axis=1)
    dist_an = _masked_min(dist_mat, is_neg, axis=1)
    return dist_ap, dist_an


class IntraDomainScatterLoss(nn.Layer):
    """
    IntraDomainScatterLoss
    
    enhance intra-domain diversity and disarrange inter-domain distributions like confusing multiple styles.

    reference: https://arxiv.org/abs/2011.14670
    """

    def __init__(self, normalize_feature, feature_from):
        super(IntraDomainScatterLoss, self).__init__()
        self.normalize_feature = normalize_feature
        self.feature_from = feature_from

    def forward(self, input, batch):
        domains = batch["domain"]
        inputs = input[self.feature_from]

        if self.normalize_feature:
            inputs = 1. * inputs / (paddle.expand_as(
                paddle.norm(
                    inputs, p=2, axis=-1, keepdim=True), inputs) + 1e-12)

        unique_label = paddle.unique(domains)
        features_per_domain = list()
        for i, x in enumerate(unique_label):
            features_per_domain.append(inputs[x == domains])
        num_domain = len(features_per_domain)
        losses = []
        for i in range(num_domain):
            features_in_same_domain = features_per_domain[i]
            center = paddle.mean(features_in_same_domain, 0)
            cos_sim = cosine_similarity(
                center.unsqueeze(0), features_in_same_domain)
            losses.append(paddle.mean(cos_sim))
        loss = paddle.mean(paddle.stack(losses))
        return {"IntraDomainScatterLoss": loss}


class InterDomainShuffleLoss(nn.Layer):
    """
    InterDomainShuffleLoss

    pull the negative sample of the interdomain and push the negative sample of the intra-domain, 
    so that the inter-domain distributions are shuffled.

    reference: https://arxiv.org/abs/2011.14670
    """

    def __init__(self, normalize_feature=True, feature_from="features"):
        super(InterDomainShuffleLoss, self).__init__()
        self.feature_from = feature_from
        self.normalize_feature = normalize_feature

    def forward(self, input, batch):
        target = batch["label"]
        domains = batch["domain"]
        inputs = input[self.feature_from]
        bs = inputs.shape[0]

        if self.normalize_feature:
            inputs = 1. * inputs / (paddle.expand_as(
                paddle.norm(
                    inputs, p=2, axis=-1, keepdim=True), inputs) + 1e-12)

        # compute distance
        dist_mat = euclidean_dist(inputs, inputs)

        is_same_img = np.zeros(shape=[bs, bs], dtype=bool)
        np.fill_diagonal(is_same_img, True)
        is_same_img = paddle.to_tensor(is_same_img)
        is_diff_instance = target.reshape([bs, 1]).expand([bs, bs])\
            .not_equal(target.reshape([bs, 1]).expand([bs, bs]).t())
        is_same_domain = domains.reshape([bs, 1]).expand([bs, bs])\
            .equal(domains.reshape([bs, 1]).expand([bs, bs]).t())
        is_diff_domain = is_same_domain == False

        is_pos = paddle.logical_or(is_same_img, is_diff_domain)
        is_neg = paddle.logical_and(is_diff_instance, is_same_domain)

        dist_ap, dist_an = hard_example_mining(dist_mat, is_pos, is_neg)

        y = paddle.ones_like(dist_an)
        loss = F.soft_margin_loss(dist_an - dist_ap, y)
        if loss == float('Inf'):
            loss = F.margin_ranking_loss(dist_an, dist_ap, y, margin=0.3)
        return {"InterDomainShuffleLoss": loss}


class CELossForMetaBIN(CELoss):
    def _labelsmoothing(self, target, class_num):
        if len(target.shape) == 1 or target.shape[-1] != class_num:
            one_hot_target = F.one_hot(target, class_num)
        else:
            one_hot_target = target
        # epsilon is different from the one in original CELoss
        epsilon = class_num / (class_num - 1) * self.epsilon
        soft_target = F.label_smooth(one_hot_target, epsilon=epsilon)
        soft_target = paddle.reshape(soft_target, shape=[-1, class_num])
        return soft_target

    def forward(self, x, batch):
        label = batch["label"]
        return super().forward(x, label)


class TripletLossForMetaBIN(nn.Layer):
    def __init__(self,
                 margin=1,
                 normalize_feature=False,
                 feature_from="feature"):
        super(TripletLossForMetaBIN, self).__init__()
        self.margin = margin
        self.feature_from = feature_from
        self.normalize_feature = normalize_feature

    def forward(self, input, batch):
        inputs = input[self.feature_from]
        targets = batch["label"]
        bs = inputs.shape[0]
        all_targets = targets

        if self.normalize_feature:
            inputs = 1. * inputs / (paddle.expand_as(
                paddle.norm(
                    inputs, p=2, axis=-1, keepdim=True), inputs) + 1e-12)

        dist_mat = euclidean_dist(inputs, inputs)

        is_pos = all_targets.reshape([bs, 1]).expand([bs, bs]).equal(
            all_targets.reshape([bs, 1]).expand([bs, bs]).t())
        is_neg = all_targets.reshape([bs, 1]).expand([bs, bs]).not_equal(
            all_targets.reshape([bs, 1]).expand([bs, bs]).t())
        dist_ap, dist_an = hard_example_mining(dist_mat, is_pos, is_neg)

        y = paddle.ones_like(dist_an)
        loss = F.margin_ranking_loss(dist_an, dist_ap, y, margin=self.margin)
        return {"TripletLoss": loss}