arrange_sample.py

# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# function:
#    operators to process sample,
#    eg: decode/resize/crop image

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import logging
import numpy as np
from .operators import BaseOperator, register_op

logger = logging.getLogger(__name__)


@register_op
class ArrangeRCNN(BaseOperator):
    """
    Transform dict to tuple format needed for training.

    Args:
        is_mask (bool): whether to use include mask data
    """

    def __init__(self, is_mask=False):
        super(ArrangeRCNN, self).__init__()
        self.is_mask = is_mask
        assert isinstance(self.is_mask, bool), "wrong type for is_mask"

    def __call__(self, sample, context=None):
        """
        Args:
            sample: a dict which contains image
                    info and annotation info.
            context: a dict which contains additional info.
        Returns:
            sample: a tuple containing following items
                (image, im_info, im_id, gt_bbox, gt_class, is_crowd, gt_masks)
        """
        im = sample['image']
        gt_bbox = sample['gt_bbox']
        gt_class = sample['gt_class']
        keys = list(sample.keys())
        if 'is_crowd' in keys:
            is_crowd = sample['is_crowd']
        else:
            raise KeyError("The dataset doesn't have 'is_crowd' key.")
        if 'im_info' in keys:
            im_info = sample['im_info']
        else:
            raise KeyError("The dataset doesn't have 'im_info' key.")
        im_id = sample['im_id']

        outs = (im, im_info, im_id, gt_bbox, gt_class, is_crowd)
        gt_masks = []
        if self.is_mask and len(sample['gt_poly']) != 0 \
                and 'is_crowd' in keys:
            valid = True
            segms = sample['gt_poly']
            assert len(segms) == is_crowd.shape[0]
            for i in range(len(sample['gt_poly'])):
                segm, iscrowd = segms[i], is_crowd[i]
                gt_segm = []
                if iscrowd:
                    gt_segm.append([[0, 0]])
                else:
                    for poly in segm:
                        if len(poly) == 0:
                            valid = False
                            break
                        gt_segm.append(np.array(poly).reshape(-1, 2))
                if (not valid) or len(gt_segm) == 0:
                    break
                gt_masks.append(gt_segm)
            outs = outs + (gt_masks, )
        return outs


@register_op
class ArrangeEvalRCNN(BaseOperator):
    """
    Transform dict to the tuple format needed for evaluation.
    """

    def __init__(self):
        super(ArrangeEvalRCNN, self).__init__()

    def __call__(self, sample, context=None):
        """
        Args:
            sample: a dict which contains image
                    info and annotation info.
            context: a dict which contains additional info.
        Returns:
            sample: a tuple containing the following items:
                    (image, im_info, im_id, im_shape, gt_bbox,
                    gt_class, difficult)
        """
        ims = []
        keys = sorted(list(sample.keys()))
        for k in keys:
            if 'image' in k:
                ims.append(sample[k])
        if 'im_info' in keys:
            im_info = sample['im_info']
        else:
            raise KeyError("The dataset doesn't have 'im_info' key.")
        im_id = sample['im_id']
        h = sample['h']
        w = sample['w']
        # For rcnn models in eval and infer stage, original image size
        # is needed to clip the bounding boxes. And box clip op in
        # bbox prediction needs im_info as input in format of [N, 3],
        # so im_shape is appended by 1 to match dimension.
        im_shape = np.array((h, w, 1), dtype=np.float32)
        gt_bbox = sample['gt_bbox']
        gt_class = sample['gt_class']
        difficult = sample['difficult']
        remain_list = [im_info, im_id, im_shape, gt_bbox, gt_class, difficult]
        ims.extend(remain_list)
        outs = tuple(ims)
        return outs


@register_op
class ArrangeTestRCNN(BaseOperator):
    """
    Transform dict to the tuple format needed for training.
    """

    def __init__(self):
        super(ArrangeTestRCNN, self).__init__()

    def __call__(self, sample, context=None):
        """
        Args:
            sample: a dict which contains image
                    info and annotation info.
            context: a dict which contains additional info.
        Returns:
            sample: a tuple containing the following items:
                    (image, im_info, im_id, im_shape)
        """
        ims = []
        keys = sorted(list(sample.keys()))
        for k in keys:
            if 'image' in k:
                ims.append(sample[k])
        if 'im_info' in keys:
            im_info = sample['im_info']
        else:
            raise KeyError("The dataset doesn't have 'im_info' key.")
        im_id = sample['im_id']
        h = sample['h']
        w = sample['w']
        # For rcnn models in eval and infer stage, original image size
        # is needed to clip the bounding boxes. And box clip op in
        # bbox prediction needs im_info as input in format of [N, 3],
        # so im_shape is appended by 1 to match dimension.
        im_shape = np.array((h, w, 1), dtype=np.float32)
        remain_list = [im_info, im_id, im_shape]
        ims.extend(remain_list)
        outs = tuple(ims)
        return outs


@register_op
class ArrangeSSD(BaseOperator):
    """
    Transform dict to tuple format needed for training.
    """

    def __init__(self):
        super(ArrangeSSD, self).__init__()

    def __call__(self, sample, context=None):
        """
        Args:
            sample: a dict which contains image
                    info and annotation info.
            context: a dict which contains additional info.
        Returns:
            sample: a tuple containing the following items:
                    (image, gt_bbox, gt_class, difficult)
        """
        im = sample['image']
        gt_bbox = sample['gt_bbox']
        gt_class = sample['gt_class']
        outs = (im, gt_bbox, gt_class)
        return outs


@register_op
class ArrangeEvalSSD(BaseOperator):
    """
    Transform dict to tuple format needed for training.
    """

    def __init__(self, fields):
        super(ArrangeEvalSSD, self).__init__()
        self.fields = fields

    def __call__(self, sample, context=None):
        """
        Args:
            sample: a dict which contains image
                    info and annotation info.
            context: a dict which contains additional info.
        Returns:
            sample: a tuple containing the following items: (image)
        """
        outs = []
        if len(sample['gt_bbox']) != len(sample['gt_class']):
            raise ValueError("gt num mismatch: bbox and class.")
        for field in self.fields:
            if field == 'im_shape':
                h = sample['h']
                w = sample['w']
                im_shape = np.array((h, w))
                outs.append(im_shape)
            elif field == 'is_difficult':
                outs.append(sample['difficult'])
            elif field == 'gt_box':
                outs.append(sample['gt_bbox'])
            elif field == 'gt_label':
                outs.append(sample['gt_class'])
            else:
                outs.append(sample[field])

        outs = tuple(outs)

        return outs


@register_op
class ArrangeTestSSD(BaseOperator):
    """
    Transform dict to tuple format needed for training.

    Args:
        is_mask (bool): whether to use include mask data
    """

    def __init__(self):
        super(ArrangeTestSSD, self).__init__()

    def __call__(self, sample, context=None):
        """
        Args:
            sample: a dict which contains image
                    info and annotation info.
            context: a dict which contains additional info.
        Returns:
            sample: a tuple containing the following items: (image)
        """
        im = sample['image']
        im_id = sample['im_id']
        h = sample['h']
        w = sample['w']
        im_shape = np.array((h, w))
        outs = (im, im_id, im_shape)
        return outs


@register_op
class ArrangeYOLO(BaseOperator):
    """
    Transform dict to the tuple format needed for training.
    """

    def __init__(self):
        super(ArrangeYOLO, self).__init__()

    def __call__(self, sample, context=None):
        """
        Args:
            sample: a dict which contains image
                    info and annotation info.
            context: a dict which contains additional info.
        Returns:
            sample: a tuple containing the following items:
                (image, gt_bbox, gt_class, gt_score,
                 is_crowd, im_info, gt_masks)
        """
        im = sample['image']
        if len(sample['gt_bbox']) != len(sample['gt_class']):
            raise ValueError("gt num mismatch: bbox and class.")
        if len(sample['gt_bbox']) != len(sample['gt_score']):
            raise ValueError("gt num mismatch: bbox and score.")
        gt_bbox = np.zeros((50, 4), dtype=im.dtype)
        gt_class = np.zeros((50, ), dtype=np.int32)
        gt_score = np.zeros((50, ), dtype=im.dtype)
        gt_num = min(50, len(sample['gt_bbox']))
        if gt_num > 0:
            gt_bbox[:gt_num, :] = sample['gt_bbox'][:gt_num, :]
            gt_class[:gt_num] = sample['gt_class'][:gt_num, 0]
            gt_score[:gt_num] = sample['gt_score'][:gt_num, 0]
        # parse [x1, y1, x2, y2] to [x, y, w, h]
        gt_bbox[:, 2:4] = gt_bbox[:, 2:4] - gt_bbox[:, :2]
        gt_bbox[:, :2] = gt_bbox[:, :2] + gt_bbox[:, 2:4] / 2.
        outs = (im, gt_bbox, gt_class, gt_score)
        return outs


@register_op
class ArrangeEvalYOLO(BaseOperator):
    """
    Transform dict to the tuple format needed for evaluation.
    """

    def __init__(self):
        super(ArrangeEvalYOLO, self).__init__()

    def __call__(self, sample, context=None):
        """
        Args:
            sample: a dict which contains image
                    info and annotation info.
            context: a dict which contains additional info.
        Returns:
            sample: a tuple containing the following items:
                (image, im_shape, im_id, gt_bbox, gt_class,
                 difficult)
        """
        im = sample['image']
        if len(sample['gt_bbox']) != len(sample['gt_class']):
            raise ValueError("gt num mismatch: bbox and class.")
        im_id = sample['im_id']
        h = sample['h']
        w = sample['w']
        im_shape = np.array((h, w))
        gt_bbox = np.zeros((50, 4), dtype=im.dtype)
        gt_class = np.zeros((50, ), dtype=np.int32)
        difficult = np.zeros((50, ), dtype=np.int32)
        gt_num = min(50, len(sample['gt_bbox']))
        if gt_num > 0:
            gt_bbox[:gt_num, :] = sample['gt_bbox'][:gt_num, :]
            gt_class[:gt_num] = sample['gt_class'][:gt_num, 0]
            difficult[:gt_num] = sample['difficult'][:gt_num, 0]
        outs = (im, im_shape, im_id, gt_bbox, gt_class, difficult)
        return outs


@register_op
class ArrangeTestYOLO(BaseOperator):
    """
    Transform dict to the tuple format needed for inference.
    """

    def __init__(self):
        super(ArrangeTestYOLO, self).__init__()

    def __call__(self, sample, context=None):
        """
        Args:
            sample: a dict which contains image
                    info and annotation info.
            context: a dict which contains additional info.
        Returns:
            sample: a tuple containing the following items:
                (image, gt_bbox, gt_class, gt_score, is_crowd,
                 im_info, gt_masks)
        """
        im = sample['image']
        im_id = sample['im_id']
        h = sample['h']
        w = sample['w']
        im_shape = np.array((h, w))
        outs = (im, im_shape, im_id)
        return outs