[MOT] video infer (#3046)

* add video infer for TestMOTReader

* format, test=documment_fix

configs/datasets/mot.yml

@@ -2,10 +2,14 @@ metric: MOT
 num_classes: 1
 
 MOTDataZoo: {
-  'MOT15_train': ['Venice-2', 'KITTI-13', 'KITTI-17', 'ETH-Bahnhof', 'ETH-Sunnyday', 'PETS09-S2L1', 'TUD-Campus', 'TUD-Stadtmitte', 'ADL-Rundle-6', 'ADL-Rundle-8', 'ETH-Pedcross2'],
+  'MOT15_train': ['ADL-Rundle-6', 'ADL-Rundle-8', 'ETH-Bahnhof', 'ETH-Pedcross2', 'ETH-Sunnyday', 'KITTI-13', 'KITTI-17', 'PETS09-S2L1', 'TUD-Campus', 'TUD-Stadtmitte', 'Venice-2'],
+  'MOT15_test': ['ADL-Rundle-1', 'ADL-Rundle-3', 'AVG-TownCentre', 'ETH-Crossing', 'ETH-Jelmoli', 'ETH-Linthescher', 'KITTI-16', 'KITTI-19', 'PETS09-S2L2', 'TUD-Crossing', 'Venice-1'],
   'MOT16_train': ['MOT16-02', 'MOT16-04', 'MOT16-05', 'MOT16-09', 'MOT16-10', 'MOT16-11', 'MOT16-13'],
+  'MOT16_test': ['MOT16-01', 'MOT16-03', 'MOT16-06', 'MOT16-07', 'MOT16-08', 'MOT16-12', 'MOT16-14'],
   'MOT17_train': ['MOT17-02-SDP', 'MOT17-04-SDP', 'MOT17-05-SDP', 'MOT17-09-SDP', 'MOT17-10-SDP', 'MOT17-11-SDP', 'MOT17-13-SDP'],
+  'MOT17_test': ['MOT17-01-SDP', 'MOT17-03-SDP', 'MOT17-06-SDP', 'MOT17-07-SDP', 'MOT17-08-SDP', 'MOT17-12-SDP', 'MOT17-14-SDP'],
   'MOT20_train': ['MOT20-01', 'MOT20-02', 'MOT20-03', 'MOT20-05'],
+  'MOT20_test': ['MOT20-04', 'MOT20-06', 'MOT20-07', 'MOT20-08'],
   'demo': ['MOT16-02'],
 }
 
@@ -36,4 +40,4 @@ EvalMOTDataset:
 TestMOTDataset:
   !MOTVideoDataset
     dataset_dir: dataset/mot
-    keep_ori_im: False
+    keep_ori_im: True # set True if save visualization images or video

configs/mot/jde/README.md

@@ -55,10 +55,12 @@ CUDA_VISIBLE_DEVICES=0 python tools/eval_mot.py -c configs/mot/jde/jde_darknet53
 Inference a vidoe in single GPU with following commands.
 
 ```bash
-# inference on video
-CUDA_VISIBLE_DEVICES=0 python tools/infer_mot.py configs/mot/jde/jde_darknet53_30e_1088x608.yml -o weights=https://paddledet.bj.bcebos.com/models/mot/jde_darknet53_30e_1088x608.pdparams --video_file={your video name}.mp4
-
+# inference on video and save a video
+CUDA_VISIBLE_DEVICES=0 python tools/infer_mot.py -c configs/mot/jde/jde_darknet53_30e_1088x608.yml -o weights=https://paddledet.bj.bcebos.com/models/mot/jde_darknet53_30e_1088x608.pdparams --video_file={your video name}.mp4 --save_videos
 ```
+**Notes:**
+ Please make sure that `ffmpeg` is installed first.
+
 ## Citations
 ```
 @article{wang2019towards,

configs/mot/jde/README_cn.md

@@ -25,7 +25,7 @@
 | DarkNet53          | 864x480 |  70.1  |  65.4  | 1341  |  6454  | 25208 |   -   |[下载链接](https://paddledet.bj.bcebos.com/models/mot/jde_darknet53_30e_864x480.pdparams) | [配置文件](https://github.com/PaddlePaddle/PaddleDetection/tree/develop/configs/mot/jde/jde_darknet53_30e_864x480.yml) |
 | DarkNet53          | 576x320 |  63.1  |  64.6  | 1357  |  7083  | 32312 |   -   |[下载链接](https://paddledet.bj.bcebos.com/models/mot/jde_darknet53_30e_576x320.pdparams) | [配置文件](https://github.com/PaddlePaddle/PaddleDetection/tree/develop/configs/mot/jde/jde_darknet53_30e_576x320.yml) |
 
-**Notes:**
+**注意:**
  JDE使用8个GPU进行训练，每个GPU上batch size为4，训练了30个epoches。
 
 ## 快速开始
@@ -52,13 +52,15 @@ CUDA_VISIBLE_DEVICES=0 python tools/eval_mot.py -c configs/mot/jde/jde_darknet53
 
 ### 3. 预测
 
-使用单个GPU过如下命令预测一个视频
+使用单个GPU通过如下命令预测一个视频，并保存为视频
 
 ```bash
 # 预测一个视频
-CUDA_VISIBLE_DEVICES=0 python tools/infer_mot.py configs/mot/jde/jde_darknet53_30e_1088x608.yml -o weights=https://paddledet.bj.bcebos.com/models/mot/jde_darknet53_30e_1088x608.pdparams --video_file={your video name}.mp4
-
+CUDA_VISIBLE_DEVICES=0 python tools/infer_mot.py -c configs/mot/jde/jde_darknet53_30e_1088x608.yml -o weights=https://paddledet.bj.bcebos.com/models/mot/jde_darknet53_30e_1088x608.pdparams --video_file={your video name}.mp4  --save_videos
 ```
+**注意:**
+ 请先确保已经安装了`ffmpeg`。
+
 ## 引用
 ```
 @article{wang2019towards,

configs/mot/jde/_base_/jde_reader_1088x608.yml

@@ -72,7 +72,6 @@ TestMOTReader:
   inputs_def:
     image_shape: [3, 608, 1088]
   sample_transforms:
-    - Decode: {}
     - LetterBoxResize: {target_size: [608, 1088]}
     - NormalizeImage: {mean: [0, 0, 0], std: [1, 1, 1], is_scale: True}
     - Permute: {}

configs/mot/jde/_base_/jde_reader_576x320.yml

@@ -72,7 +72,6 @@ TestMOTReader:
   inputs_def:
     image_shape: [3, 320, 576]
   sample_transforms:
-    - Decode: {}
     - LetterBoxResize: {target_size: [320, 576]}
     - NormalizeImage: {mean: [0, 0, 0], std: [1, 1, 1], is_scale: True}
     - Permute: {}

configs/mot/jde/_base_/jde_reader_864x480.yml

@@ -72,7 +72,6 @@ TestMOTReader:
   inputs_def:
     image_shape: [3, 480, 864]
   sample_transforms:
-    - Decode: {}
     - LetterBoxResize: {target_size: [480, 864]}
     - NormalizeImage: {mean: [0, 0, 0], std: [1, 1, 1], is_scale: True}
     - Permute: {}

ppdet/data/transform/mot_operators.py

@@ -35,8 +35,8 @@ from ppdet.utils.logger import setup_logger
 logger = setup_logger(__name__)
 
 __all__ = [
-    'LetterBoxResize', 'Gt2JDETargetThres', 'Gt2JDETargetMax',
-    'Gt2FairMOTTarget'
+    'LetterBoxResize', 'MOTRandomAffine', 'Gt2JDETargetThres',
+    'Gt2JDETargetMax', 'Gt2FairMOTTarget'
 ]
 
 
@@ -113,6 +113,138 @@ class LetterBoxResize(BaseOperator):
         return sample
 
 
+@register_op
+class MOTRandomAffine(BaseOperator):
+    """ 
+    Affine transform to image and coords to achieve the rotate, scale and
+    shift effect for training image.
+
+    Args:
+        degrees (list[2]): the rotate range to apply, transform range is [min, max]
+        translate (list[2]): the translate range to apply, ransform range is [min, max]
+        scale (list[2]): the scale range to apply, transform range is [min, max]
+        shear (list[2]): the shear range to apply, transform range is [min, max]
+        borderValue (list[3]): value used in case of a constant border when appling
+            the perspective transformation
+        reject_outside (bool): reject warped bounding bboxes outside of image
+
+    Returns:
+        records(dict): contain the image and coords after tranformed
+
+    """
+
+    def __init__(self,
+                 degrees=(-5, 5),
+                 translate=(0.10, 0.10),
+                 scale=(0.50, 1.20),
+                 shear=(-2, 2),
+                 borderValue=(127.5, 127.5, 127.5),
+                 reject_outside=True):
+        super(MOTRandomAffine, self).__init__()
+        self.degrees = degrees
+        self.translate = translate
+        self.scale = scale
+        self.shear = shear
+        self.borderValue = borderValue
+        self.reject_outside = reject_outside
+
+    def apply(self, sample, context=None):
+        # https://medium.com/uruvideo/dataset-augmentation-with-random-homographies-a8f4b44830d4
+        border = 0  # width of added border (optional)
+
+        img = sample['image']
+        height, width = img.shape[0], img.shape[1]
+
+        # Rotation and Scale
+        R = np.eye(3)
+        a = random.random() * (self.degrees[1] - self.degrees[0]
+                               ) + self.degrees[0]
+        s = random.random() * (self.scale[1] - self.scale[0]) + self.scale[0]
+        R[:2] = cv2.getRotationMatrix2D(
+            angle=a, center=(width / 2, height / 2), scale=s)
+
+        # Translation
+        T = np.eye(3)
+        T[0, 2] = (
+            random.random() * 2 - 1
+        ) * self.translate[0] * height + border  # x translation (pixels)
+        T[1, 2] = (
+            random.random() * 2 - 1
+        ) * self.translate[1] * width + border  # y translation (pixels)
+
+        # Shear
+        S = np.eye(3)
+        S[0, 1] = math.tan((random.random() *
+                            (self.shear[1] - self.shear[0]) + self.shear[0]) *
+                           math.pi / 180)  # x shear (deg)
+        S[1, 0] = math.tan((random.random() *
+                            (self.shear[1] - self.shear[0]) + self.shear[0]) *
+                           math.pi / 180)  # y shear (deg)
+
+        M = S @T @R  # Combined rotation matrix. ORDER IS IMPORTANT HERE!!
+        imw = cv2.warpPerspective(
+            img,
+            M,
+            dsize=(width, height),
+            flags=cv2.INTER_LINEAR,
+            borderValue=self.borderValue)  # BGR order borderValue
+
+        if 'gt_bbox' in sample and len(sample['gt_bbox']) > 0:
+            targets = sample['gt_bbox']
+            n = targets.shape[0]
+            points = targets.copy()
+            area0 = (points[:, 2] - points[:, 0]) * (
+                points[:, 3] - points[:, 1])
+
+            # warp points
+            xy = np.ones((n * 4, 3))
+            xy[:, :2] = points[:, [0, 1, 2, 3, 0, 3, 2, 1]].reshape(
+                n * 4, 2)  # x1y1, x2y2, x1y2, x2y1
+            xy = (xy @M.T)[:, :2].reshape(n, 8)
+
+            # create new boxes
+            x = xy[:, [0, 2, 4, 6]]
+            y = xy[:, [1, 3, 5, 7]]
+            xy = np.concatenate(
+                (x.min(1), y.min(1), x.max(1), y.max(1))).reshape(4, n).T
+
+            # apply angle-based reduction
+            radians = a * math.pi / 180
+            reduction = max(abs(math.sin(radians)), abs(math.cos(radians)))**0.5
+            x = (xy[:, 2] + xy[:, 0]) / 2
+            y = (xy[:, 3] + xy[:, 1]) / 2
+            w = (xy[:, 2] - xy[:, 0]) * reduction
+            h = (xy[:, 3] - xy[:, 1]) * reduction
+            xy = np.concatenate(
+                (x - w / 2, y - h / 2, x + w / 2, y + h / 2)).reshape(4, n).T
+
+            # reject warped points outside of image
+            if self.reject_outside:
+                np.clip(xy[:, 0], 0, width, out=xy[:, 0])
+                np.clip(xy[:, 2], 0, width, out=xy[:, 2])
+                np.clip(xy[:, 1], 0, height, out=xy[:, 1])
+                np.clip(xy[:, 3], 0, height, out=xy[:, 3])
+            w = xy[:, 2] - xy[:, 0]
+            h = xy[:, 3] - xy[:, 1]
+            area = w * h
+            ar = np.maximum(w / (h + 1e-16), h / (w + 1e-16))
+            i = (w > 4) & (h > 4) & (area / (area0 + 1e-16) > 0.1) & (ar < 10)
+
+            if sum(i) > 0:
+                sample['gt_bbox'] = xy[i].astype(sample['gt_bbox'].dtype)
+                sample['gt_class'] = sample['gt_class'][i]
+                if 'difficult' in sample:
+                    sample['difficult'] = sample['difficult'][i]
+                if 'gt_ide' in sample:
+                    sample['gt_ide'] = sample['gt_ide'][i]
+                if 'is_crowd' in sample:
+                    sample['is_crowd'] = sample['is_crowd'][i]
+                sample['image'] = imw
+                return sample
+            else:
+                return sample
+
+
 @register_op
 class Gt2JDETargetThres(BaseOperator):
     __shared__ = ['num_classes']

ppdet/data/transform/operators.py

@@ -2081,139 +2081,6 @@ class Norm2PixelBbox(BaseOperator):
         return sample
 
 
-@register_op
-class MOTRandomAffine(BaseOperator):
-    """ 
-    Affine transform to image and coords to achieve the rotate, scale and
-    shift effect for training image.
-
-    Args:
-        degrees (list[2]): the rotate range to apply, transform range is [min, max]
-        translate (list[2]): the translate range to apply, ransform range is [min, max]
-        scale (list[2]): the scale range to apply, transform range is [min, max]
-        shear (list[2]): the shear range to apply, transform range is [min, max]
-        borderValue (list[3]): value used in case of a constant border when appling
-            the perspective transformation
-        reject_outside (bool): reject warped bounding bboxes outside of image
-
-    Returns:
-        records(dict): contain the image and coords after tranformed
-
-    """
-
-    def __init__(self,
-                 degrees=(-5, 5),
-                 translate=(0.10, 0.10),
-                 scale=(0.50, 1.20),
-                 shear=(-2, 2),
-                 borderValue=(127.5, 127.5, 127.5),
-                 reject_outside=True):
-
-        super(MOTRandomAffine, self).__init__()
-        self.degrees = degrees
-        self.translate = translate
-        self.scale = scale
-        self.shear = shear
-        self.borderValue = borderValue
-        self.reject_outside = reject_outside
-
-    def apply(self, sample, context=None):
-        # https://medium.com/uruvideo/dataset-augmentation-with-random-homographies-a8f4b44830d4
-        border = 0  # width of added border (optional)
-
-        img = sample['image']
-        height, width = img.shape[0], img.shape[1]
-
-        # Rotation and Scale
-        R = np.eye(3)
-        a = random.random() * (self.degrees[1] - self.degrees[0]
-                               ) + self.degrees[0]
-        s = random.random() * (self.scale[1] - self.scale[0]) + self.scale[0]
-        R[:2] = cv2.getRotationMatrix2D(
-            angle=a, center=(width / 2, height / 2), scale=s)
-
-        # Translation
-        T = np.eye(3)
-        T[0, 2] = (
-            random.random() * 2 - 1
-        ) * self.translate[0] * height + border  # x translation (pixels)
-        T[1, 2] = (
-            random.random() * 2 - 1
-        ) * self.translate[1] * width + border  # y translation (pixels)
-
-        # Shear
-        S = np.eye(3)
-        S[0, 1] = math.tan((random.random() *
-                            (self.shear[1] - self.shear[0]) + self.shear[0]) *
-                           math.pi / 180)  # x shear (deg)
-        S[1, 0] = math.tan((random.random() *
-                            (self.shear[1] - self.shear[0]) + self.shear[0]) *
-                           math.pi / 180)  # y shear (deg)
-
-        M = S @T @R  # Combined rotation matrix. ORDER IS IMPORTANT HERE!!
-        imw = cv2.warpPerspective(
-            img,
-            M,
-            dsize=(width, height),
-            flags=cv2.INTER_LINEAR,
-            borderValue=self.borderValue)  # BGR order borderValue
-
-        if 'gt_bbox' in sample and len(sample['gt_bbox']) > 0:
-            targets = sample['gt_bbox']
-            n = targets.shape[0]
-            points = targets.copy()
-            area0 = (points[:, 2] - points[:, 0]) * (
-                points[:, 3] - points[:, 1])
-
-            # warp points
-            xy = np.ones((n * 4, 3))
-            xy[:, :2] = points[:, [0, 1, 2, 3, 0, 3, 2, 1]].reshape(
-                n * 4, 2)  # x1y1, x2y2, x1y2, x2y1
-            xy = (xy @M.T)[:, :2].reshape(n, 8)
-
-            # create new boxes
-            x = xy[:, [0, 2, 4, 6]]
-            y = xy[:, [1, 3, 5, 7]]
-            xy = np.concatenate(
-                (x.min(1), y.min(1), x.max(1), y.max(1))).reshape(4, n).T
-
-            # apply angle-based reduction
-            radians = a * math.pi / 180
-            reduction = max(abs(math.sin(radians)), abs(math.cos(radians)))**0.5
-            x = (xy[:, 2] + xy[:, 0]) / 2
-            y = (xy[:, 3] + xy[:, 1]) / 2
-            w = (xy[:, 2] - xy[:, 0]) * reduction
-            h = (xy[:, 3] - xy[:, 1]) * reduction
-            xy = np.concatenate(
-                (x - w / 2, y - h / 2, x + w / 2, y + h / 2)).reshape(4, n).T
-
-            # reject warped points outside of image
-            if self.reject_outside:
-                np.clip(xy[:, 0], 0, width, out=xy[:, 0])
-                np.clip(xy[:, 2], 0, width, out=xy[:, 2])
-                np.clip(xy[:, 1], 0, height, out=xy[:, 1])
-                np.clip(xy[:, 3], 0, height, out=xy[:, 3])
-            w = xy[:, 2] - xy[:, 0]
-            h = xy[:, 3] - xy[:, 1]
-            area = w * h
-            ar = np.maximum(w / (h + 1e-16), h / (w + 1e-16))
-            i = (w > 4) & (h > 4) & (area / (area0 + 1e-16) > 0.1) & (ar < 10)
-
-            if sum(i) > 0:
-                sample['gt_bbox'] = xy[i].astype(sample['gt_bbox'].dtype)
-                sample['gt_class'] = sample['gt_class'][i]
-                if 'difficult' in sample:
-                    sample['difficult'] = sample['difficult'][i]
-                if 'gt_ide' in sample:
-                    sample['gt_ide'] = sample['gt_ide'][i]
-                if 'is_crowd' in sample:
-                    sample['is_crowd'] = sample['is_crowd'][i]
-                sample['image'] = imw
-                return sample
-            else:
-                return sample
-
-
 @register_op
 class BboxCXCYWH2XYXY(BaseOperator):
     """