diff --git a/ppdet/data/transform/op_helper.py b/ppdet/data/transform/op_helper.py index 838714f4dda2b664ae4d2b1f3ee343e5b6e50360..994f70d707e842bdc7cd9aa903cf92e149419d6f 100644 --- a/ppdet/data/transform/op_helper.py +++ b/ppdet/data/transform/op_helper.py @@ -387,3 +387,9 @@ def crop_image_sampling(img, sample_bbox, image_width, image_height, sample_img, (target_size, target_size), interpolation=cv2.INTER_AREA) return sample_img + + +def is_poly(segm): + assert isinstance(segm, (list, dict)), \ + "Invalid segm type: {}".format(type(segm)) + return isinstance(segm, list) diff --git a/ppdet/data/transform/operators.py b/ppdet/data/transform/operators.py index 5b80f88a7f9bc9a6b54c025d390cc26477c37208..dbe8a0c3fffbd4e1cc4da8714ddb104dfdcb6c83 100644 --- a/ppdet/data/transform/operators.py +++ b/ppdet/data/transform/operators.py @@ -41,7 +41,8 @@ from ppdet.core.workspace import serializable from .op_helper import (satisfy_sample_constraint, filter_and_process, generate_sample_bbox, clip_bbox, data_anchor_sampling, satisfy_sample_constraint_coverage, crop_image_sampling, - generate_sample_bbox_square, bbox_area_sampling) + generate_sample_bbox_square, bbox_area_sampling, + is_poly) logger = logging.getLogger(__name__) @@ -360,17 +361,12 @@ class RandomFlipImage(BaseOperator): def _flip_rle(rle, height, width): if 'counts' in rle and type(rle['counts']) == list: - rle = mask_util.frPyObjects([rle], height, width) + rle = mask_util.frPyObjects(rle, height, width) mask = mask_util.decode(rle) - mask = mask[:, ::-1, :] + mask = mask[:, ::-1] rle = mask_util.encode(np.array(mask, order='F', dtype=np.uint8)) return rle - def is_poly(segm): - assert isinstance(segm, (list, dict)), \ - "Invalid segm type: {}".format(type(segm)) - return isinstance(segm, list) - flipped_segms = [] for segm in segms: if is_poly(segm): @@ -1317,6 +1313,38 @@ class RandomExpand(BaseOperator): fill_value = tuple(fill_value) self.fill_value = fill_value + def expand_segms(self, segms, x, y, height, width, ratio): + def _expand_poly(poly, x, y): + expanded_poly = np.array(poly) + expanded_poly[0::2] += x + expanded_poly[1::2] += y + return expanded_poly.tolist() + + def _expand_rle(rle, x, y, height, width, ratio): + if 'counts' in rle and type(rle['counts']) == list: + rle = mask_util.frPyObjects(rle, height, width) + mask = mask_util.decode(rle) + expanded_mask = np.full((int(height * ratio), int(width * ratio)), + 0).astype(mask.dtype) + expanded_mask[y:y + height, x:x + width] = mask + rle = mask_util.encode( + np.array( + expanded_mask, order='F', dtype=np.uint8)) + return rle + + expanded_segms = [] + for segm in segms: + if is_poly(segm): + # Polygon format + expanded_segms.append( + [_expand_poly(poly, x, y) for poly in segm]) + else: + # RLE format + import pycocotools.mask as mask_util + expanded_segms.append( + _expand_rle(segm, x, y, height, width, ratio)) + return expanded_segms + def __call__(self, sample, context=None): if np.random.uniform(0., 1.) < self.prob: return sample @@ -1341,7 +1369,9 @@ class RandomExpand(BaseOperator): sample['image'] = canvas if 'gt_bbox' in sample and len(sample['gt_bbox']) > 0: sample['gt_bbox'] += np.array([x, y] * 2, dtype=np.float32) - + if 'gt_poly' in sample and len(sample['gt_poly']) > 0: + sample['gt_poly'] = self.expand_segms(sample['gt_poly'], x, y, + height, width, expand_ratio) return sample @@ -1375,6 +1405,75 @@ class RandomCrop(BaseOperator): self.allow_no_crop = allow_no_crop self.cover_all_box = cover_all_box + def crop_segms(self, segms, valid_ids, crop, height, width): + def _crop_poly(segm, crop): + xmin, ymin, xmax, ymax = crop + crop_coord = [xmin, ymin, xmin, ymax, xmax, ymax, xmax, ymin] + crop_p = np.array(crop_coord).reshape(4, 2) + crop_p = Polygon(crop_p) + + crop_segm = list() + for poly in segm: + poly = np.array(poly).reshape(len(poly) // 2, 2) + polygon = Polygon(poly) + if not polygon.is_valid: + exterior = polygon.exterior + multi_lines = exterior.intersection(exterior) + polygons = shapely.ops.polygonize(multi_lines) + polygon = MultiPolygon(polygons) + multi_polygon = list() + if isinstance(polygon, MultiPolygon): + multi_polygon = copy.deepcopy(polygon) + else: + multi_polygon.append(copy.deepcopy(polygon)) + for per_polygon in multi_polygon: + inter = per_polygon.intersection(crop_p) + if not inter: + continue + if isinstance(inter, (MultiPolygon, GeometryCollection)): + for part in inter: + if not isinstance(part, Polygon): + continue + part = np.squeeze( + np.array(part.exterior.coords[:-1]).reshape(1, + -1)) + part[0::2] -= xmin + part[1::2] -= ymin + crop_segm.append(part.tolist()) + elif isinstance(inter, Polygon): + crop_poly = np.squeeze( + np.array(inter.exterior.coords[:-1]).reshape(1, -1)) + crop_poly[0::2] -= xmin + crop_poly[1::2] -= ymin + crop_segm.append(crop_poly.tolist()) + else: + continue + return crop_segm + + def _crop_rle(rle, crop, height, width): + if 'counts' in rle and type(rle['counts']) == list: + rle = mask_util.frPyObjects(rle, height, width) + mask = mask_util.decode(rle) + mask = mask[crop[1]:crop[3], crop[0]:crop[2]] + rle = mask_util.encode(np.array(mask, order='F', dtype=np.uint8)) + return rle + + crop_segms = [] + for id in valid_ids: + segm = segms[id] + if is_poly(segm): + import copy + import shapely.ops + from shapely.geometry import Polygon, MultiPolygon, GeometryCollection + logging.getLogger("shapely").setLevel(logging.WARNING) + # Polygon format + crop_segms.append(_crop_poly(segm, crop)) + else: + # RLE format + import pycocotools.mask as mask_util + crop_segms.append(_crop_rle(segm, crop, height, width)) + return crop_segms + def __call__(self, sample, context=None): if 'gt_bbox' in sample and len(sample['gt_bbox']) == 0: return sample @@ -1428,6 +1527,28 @@ class RandomCrop(BaseOperator): break if found: + if 'gt_poly' in sample and len(sample['gt_poly']) > 0: + crop_polys = self.crop_segms( + sample['gt_poly'], + valid_ids, + np.array( + crop_box, dtype=np.int64), + h, + w) + if [] in crop_polys: + delete_id = list() + valid_polys = list() + for id, crop_poly in enumerate(crop_polys): + if crop_poly == []: + delete_id.append(id) + else: + valid_polys.append(crop_poly) + valid_ids = np.delete(valid_ids, delete_id) + if len(valid_polys) == 0: + return sample + sample['gt_poly'] = valid_polys + else: + sample['gt_poly'] = crop_polys sample['image'] = self._crop_image(sample['image'], crop_box) sample['gt_bbox'] = np.take(cropped_box, valid_ids, axis=0) sample['gt_class'] = np.take( @@ -1437,6 +1558,10 @@ class RandomCrop(BaseOperator): if 'gt_score' in sample: sample['gt_score'] = np.take( sample['gt_score'], valid_ids, axis=0) + + if 'is_crowd' in sample: + sample['is_crowd'] = np.take( + sample['is_crowd'], valid_ids, axis=0) return sample return sample