add md5sum check and illegal parking recognition (#6608)

* 1. add md5sum check;
2. add illegal parking recognition.

* add config for illegal parking recognition

* car licenseplate skipframe&time counter vote (#6609)

* car licenseplate skipframe&time counter vote

* incase of no track

* update the method to get plate

* if illegal_parking_time is -1, do not update_object_info

* get model dir from cfg

* illegal parking visualize optimize
Co-authored-by: Nzhiboniu <31800336+zhiboniu@users.noreply.github.com>

deploy/pipeline/cfg_utils.py

@@ -127,6 +127,12 @@ def argsparser():
         help="Whether counting the numbers of identifiers break in "
         "the area. Note that only support single-class MOT and "
         "the video should be taken by a static camera.")
+    parser.add_argument(
+        "--illegal_parking_time",
+        type=int,
+        default=-1,
+        help="illegal parking time which units are seconds, default is -1 which means not recognition illegal parking"
+    )
     parser.add_argument(
         "--region_type",
         type=str,

deploy/pipeline/config/examples/infer_cfg_illegal_parking.yml

+crop_thresh: 0.5
+visual: True
+warmup_frame: 50
+
+MOT:
+  model_dir: https://bj.bcebos.com/v1/paddledet/models/pipeline/mot_ppyoloe_l_36e_ppvehicle.zip
+  tracker_config: deploy/pipeline/config/tracker_config.yml
+  batch_size: 1
+  enable: True
+
+VEHICLE_PLATE:
+  det_model_dir: https://paddleocr.bj.bcebos.com/PP-OCRv3/chinese/ch_PP-OCRv3_det_infer.tar
+  det_limit_side_len: 480
+  det_limit_type: "max"
+  rec_model_dir: https://paddleocr.bj.bcebos.com/PP-OCRv3/chinese/ch_PP-OCRv3_rec_infer.tar
+  rec_image_shape: [3, 48, 320]
+  rec_batch_num: 6
+  word_dict_path: deploy/pipeline/ppvehicle/rec_word_dict.txt
+  enable: True

deploy/pipeline/download.py

@@ -29,6 +29,31 @@ DOWNLOAD_RETRY_LIMIT = 3
 
 WEIGHTS_HOME = osp.expanduser("~/.cache/paddle/infer_weights")
 
+MODEL_URL_MD5_DICT = {
+    "https://paddleocr.bj.bcebos.com/PP-OCRv3/chinese/ch_PP-OCRv3_det_infer.tar":
+    '982d2d8d83e55f5f981e96a7b941fff5',
+    "https://paddleocr.bj.bcebos.com/PP-OCRv3/chinese/ch_PP-OCRv3_rec_infer.tar":
+    '5f021b88518bdeda2cb4a3aacc481024',
+    "https://bj.bcebos.com/v1/paddledet/models/pipeline/mot_ppyoloe_l_36e_ppvehicle.zip":
+    "3859d1a26e0c498285c2374b1a347013",
+    "https://bj.bcebos.com/v1/paddledet/models/pipeline/mot_ppyoloe_l_36e_ppvehicle.zip":
+    "46a80e1b3a8f4599e0cc79367c195b7c",
+    "https://bj.bcebos.com/v1/paddledet/models/pipeline/dark_hrnet_w32_256x192.zip":
+    "a20d5f6ca087bff0e9f2b18df45a36f2",
+    "https://bj.bcebos.com/v1/paddledet/models/pipeline/PPLCNet_x1_0_person_attribute_945_infer.zip":
+    "1dfb161bf12bbc1365b2ed6866674483",
+    "https://videotag.bj.bcebos.com/PaddleVideo-release2.3/ppTSM_fight.zip":
+    "5d4609142501258608bf0a1445eedaba",
+    "https://bj.bcebos.com/v1/paddledet/models/pipeline/STGCN.zip":
+    "cf1c3c4bae90b975accb954d13129ea4",
+    "https://bj.bcebos.com/v1/paddledet/models/pipeline/ppyoloe_crn_s_80e_smoking_visdrone.zip":
+    "4cd12ae55be8f0eb2b90c08ac3b48218",
+    "https://bj.bcebos.com/v1/paddledet/models/pipeline/PPHGNet_tiny_calling_halfbody.zip":
+    "cf86b87ace97540dace6ef08e62b584a",
+    "https://bj.bcebos.com/v1/paddledet/models/pipeline/reid_model.zip":
+    "fdc4dac38393b8e2b5921c1e1fdd5315"
+}
+
 
 def is_url(path):
     """
@@ -91,7 +116,6 @@ def _download(url, path, md5sum=None):
     fname = osp.split(url)[-1]
     fullname = osp.join(path, fname)
     retry_cnt = 0
-
     while not (osp.exists(fullname) and _check_exist_file_md5(fullname, md5sum,
                                                               url)):
         if retry_cnt < DOWNLOAD_RETRY_LIMIT:
@@ -296,7 +320,10 @@ def get_weights_path(url):
     download it from url.
     """
     url = parse_url(url)
-    path, _ = get_path(url, WEIGHTS_HOME)
+    md5sum = None
+    if url in MODEL_URL_MD5_DICT.keys():
+        md5sum = MODEL_URL_MD5_DICT[url]
+    path, _ = get_path(url, WEIGHTS_HOME, md5sum)
     return path
 
 

deploy/pipeline/pipeline.py

@@ -40,7 +40,7 @@ from python.visualize import visualize_box_mask, visualize_attr, visualize_pose,
 
 from pptracking.python.mot_sde_infer import SDE_Detector
 from pptracking.python.mot.visualize import plot_tracking_dict
-from pptracking.python.mot.utils import flow_statistic
+from pptracking.python.mot.utils import flow_statistic, update_object_info
 
 from pphuman.attr_infer import AttrDetector
 from pphuman.video_action_infer import VideoActionRecognizer
@@ -283,6 +283,7 @@ class PipePredictor(object):
         self.is_video = is_video
         self.multi_camera = multi_camera
         self.cfg = cfg
+
         self.output_dir = args.output_dir
         self.draw_center_traj = args.draw_center_traj
         self.secs_interval = args.secs_interval
@@ -290,6 +291,7 @@ class PipePredictor(object):
         self.do_break_in_counting = args.do_break_in_counting
         self.region_type = args.region_type
         self.region_polygon = args.region_polygon
+        self.illegal_parking_time = args.illegal_parking_time
 
         self.warmup_frame = self.cfg['warmup_frame']
         self.pipeline_res = Result()
@@ -376,6 +378,13 @@ class PipePredictor(object):
                     use_dark=False)
                 self.kpt_buff = KeyPointBuff(skeleton_action_frames)
 
+            if self.with_vehicleplate:
+                vehicleplate_cfg = self.cfg['VEHICLE_PLATE']
+                self.vehicleplate_detector = PlateRecognizer(args,
+                                                             vehicleplate_cfg)
+                basemode = self.basemode['VEHICLE_PLATE']
+                self.modebase[basemode] = True
+
             if self.with_mtmct:
                 reid_cfg = self.cfg['REID']
                 basemode = self.basemode['REID']
@@ -545,7 +554,7 @@ class PipePredictor(object):
         out_id_list = list()
         prev_center = dict()
         records = list()
-        if self.do_entrance_counting or self.do_break_in_counting:
+        if self.do_entrance_counting or self.do_break_in_counting or self.illegal_parking_time != -1:
             if self.region_type == 'horizontal':
                 entrance = [0, height / 2., width, height / 2.]
             elif self.region_type == 'vertical':
@@ -575,6 +584,10 @@ class PipePredictor(object):
             short_size = self.cfg["VIDEO_ACTION"]["short_size"]
             scale = ShortSizeScale(short_size)
 
+        object_in_region_info = {
+        }  # store info for vehicle parking in region       
+        illegal_parking_dict = None
+
         while (1):
             if frame_id % 10 == 0:
                 print('frame id: ', frame_id)
@@ -608,6 +621,16 @@ class PipePredictor(object):
                     prev_center, records)
                 records = statistic['records']
 
+                if self.illegal_parking_time != -1:
+                    object_in_region_info, illegal_parking_dict = update_object_info(
+                        object_in_region_info, mot_result, self.region_type,
+                        entrance, video_fps, self.illegal_parking_time)
+                    if len(illegal_parking_dict) != 0:
+                        # build relationship between id and plate
+                        for key, value in illegal_parking_dict.items():
+                            plate = self.collector.get_carlp(key)
+                            illegal_parking_dict[key]['plate'] = plate
+
                 # nothing detected
                 if len(mot_res['boxes']) == 0:
                     frame_id += 1
@@ -786,9 +809,12 @@ class PipePredictor(object):
 
             if self.cfg['visual']:
                 _, _, fps = self.pipe_timer.get_total_time()
-                im = self.visualize_video(
-                    frame, self.pipeline_res, self.collector, frame_id, fps,
-                    entrance, records, center_traj)  # visualize
+
+                im = self.visualize_video(frame, self.pipeline_res,
+                                          self.collector, frame_id, fps,
+                                          entrance, records, center_traj,
+                                          self.illegal_parking_time != -1,
+                                          illegal_parking_dict)  # visualize
                 writer.write(im)
                 if self.file_name is None:  # use camera_id
                     cv2.imshow('Paddle-Pipeline', im)
@@ -806,7 +832,9 @@ class PipePredictor(object):
                         fps,
                         entrance=None,
                         records=None,
-                        center_traj=None):
+                        center_traj=None,
+                        do_illegal_parking_recognition=False,
+                        illegal_parking_dict=None):
         mot_res = copy.deepcopy(result.get('mot'))
         if mot_res is not None:
             ids = mot_res['boxes'][:, 0]
@@ -840,6 +868,8 @@ class PipePredictor(object):
                 ids2names=self.mot_predictor.pred_config.labels,
                 do_entrance_counting=self.do_entrance_counting,
                 do_break_in_counting=self.do_break_in_counting,
+                do_illegal_parking_recognition=do_illegal_parking_recognition,
+                illegal_parking_dict=illegal_parking_dict,
                 entrance=entrance,
                 records=records,
                 center_traj=center_traj)

deploy/pptracking/python/mot/utils.py

@@ -17,10 +17,12 @@ import cv2
 import time
 import numpy as np
 import collections
+import math
 
 __all__ = [
     'MOTTimer', 'Detection', 'write_mot_results', 'load_det_results',
-    'preprocess_reid', 'get_crops', 'clip_box', 'scale_coords', 'flow_statistic'
+    'preprocess_reid', 'get_crops', 'clip_box', 'scale_coords',
+    'flow_statistic', 'update_object_info'
 ]
 
 
@@ -335,6 +337,93 @@ def flow_statistic(result,
     }
 
 
+def distance(center_1, center_2):
+    return math.sqrt(
+        math.pow(center_1[0] - center_2[0], 2) + math.pow(center_1[1] -
+                                                          center_2[1], 2))
+
+
+# update vehicle parking info
+def update_object_info(object_in_region_info,
+                       result,
+                       region_type,
+                       entrance,
+                       fps,
+                       illegal_parking_time,
+                       distance_threshold_frame=3,
+                       distance_threshold_interval=50):
+    '''
+    For consecutive frames, the distance between two frame is smaller than distance_threshold_frame, regard as parking
+    For parking in general, the move distance should smaller than distance_threshold_interval
+    The moving distance of the vehicle is scaled according to the y, which is inversely proportional to y.
+    '''
+
+    assert region_type in [
+        'custom'
+    ], "region_type should be 'custom' when do break_in counting."
+    assert len(
+        entrance
+    ) >= 4, "entrance should be at least 3 points and (w,h) of image when do break_in counting."
+
+    frame_id, tlwhs, tscores, track_ids = result  # result from mot
+
+    im_w, im_h = entrance[-1][:]
+    entrance = np.array(entrance[:-1])
+
+    illegal_parking_dict = {}
+    for tlwh, score, track_id in zip(tlwhs, tscores, track_ids):
+        if track_id < 0: continue
+
+        x1, y1, w, h = tlwh
+        center_x = min(x1 + w / 2., im_w - 1)
+        center_y = min(y1 + h / 2, im_h - 1)
+
+        if not in_quadrangle([center_x, center_y], entrance, im_h, im_w):
+            continue
+
+        current_center = (center_x, center_y)
+        if track_id not in object_in_region_info.keys(
+        ):  # first time appear in region
+            object_in_region_info[track_id] = {}
+            object_in_region_info[track_id]["start_frame"] = frame_id
+            object_in_region_info[track_id]["end_frame"] = frame_id
+            object_in_region_info[track_id]["prev_center"] = current_center
+            object_in_region_info[track_id]["start_center"] = current_center
+        else:
+            prev_center = object_in_region_info[track_id]["prev_center"]
+
+            dis = distance(current_center, prev_center)
+            scaled_dis = 200 * dis / (
+                current_center[1] + 1)  # scale distance according to y
+            dis = scaled_dis
+
+            if dis < distance_threshold_frame:  # not move
+                object_in_region_info[track_id]["end_frame"] = frame_id
+                object_in_region_info[track_id]["prev_center"] = current_center
+            else:  # move
+                object_in_region_info[track_id]["start_frame"] = frame_id
+                object_in_region_info[track_id]["end_frame"] = frame_id
+                object_in_region_info[track_id]["prev_center"] = current_center
+                object_in_region_info[track_id]["start_center"] = current_center
+
+        # whether current object parking
+        distance_from_start = distance(
+            object_in_region_info[track_id]["start_center"], current_center)
+        if distance_from_start > distance_threshold_interval:
+            # moved
+            object_in_region_info[track_id]["start_frame"] = frame_id
+            object_in_region_info[track_id]["end_frame"] = frame_id
+            object_in_region_info[track_id]["prev_center"] = current_center
+            object_in_region_info[track_id]["start_center"] = current_center
+            continue
+
+        if (object_in_region_info[track_id]["end_frame"]-object_in_region_info[track_id]["start_frame"]) /fps >= illegal_parking_time \
+            and distance_from_start<distance_threshold_interval:
+            illegal_parking_dict[track_id] = {"bbox": [x1, y1, w, h]}
+
+    return object_in_region_info, illegal_parking_dict
+
+
 def in_quadrangle(point, entrance, im_h, im_w):
     mask = np.zeros((im_h, im_w, 1), np.uint8)
     cv2.fillPoly(mask, [entrance], 255)

deploy/pptracking/python/mot/visualize.py

@@ -194,12 +194,14 @@ def plot_tracking_dict(image,
                        ids2names=[],
                        do_entrance_counting=False,
                        do_break_in_counting=False,
+                       do_illegal_parking_recognition=False,
+                       illegal_parking_dict=None,
                        entrance=None,
                        records=None,
                        center_traj=None):
     im = np.ascontiguousarray(np.copy(image))
     im_h, im_w = im.shape[:2]
-    if do_break_in_counting:
+    if do_break_in_counting or do_illegal_parking_recognition:
         entrance = np.array(entrance[:-1])  # last pair is [im_w, im_h] 
 
     text_scale = max(0.5, image.shape[1] / 3000.)
@@ -234,7 +236,8 @@ def plot_tracking_dict(image,
             text_scale, (0, 0, 255),
             thickness=text_thickness)
 
-    if num_classes == 1 and do_break_in_counting:
+    if num_classes == 1 and (do_break_in_counting or
+                             do_illegal_parking_recognition):
         np_masks = np.zeros((im_h, im_w, 1), np.uint8)
         cv2.fillPoly(np_masks, [entrance], 255)
 
@@ -249,14 +252,33 @@ def plot_tracking_dict(image,
         im[idx[0], idx[1], :] += alpha * color_mask
         im = np.array(im).astype('uint8')
 
-        # find start location for break in counting data
-        start = records[-1].find('Break_in')
-        cv2.putText(
-            im,
-            records[-1][start:-1], (entrance[0][0] - 10, entrance[0][1] - 10),
-            cv2.FONT_ITALIC,
-            text_scale, (0, 0, 255),
-            thickness=text_thickness)
+        if do_break_in_counting:
+            # find start location for break in counting data
+            start = records[-1].find('Break_in')
+            cv2.putText(
+                im,
+                records[-1][start:-1],
+                (entrance[0][0] - 10, entrance[0][1] - 10),
+                cv2.FONT_ITALIC,
+                text_scale, (0, 0, 255),
+                thickness=text_thickness)
+
+        if illegal_parking_dict is not None and len(illegal_parking_dict) != 0:
+            for key, value in illegal_parking_dict.items():
+                x1, y1, w, h = value['bbox']
+                plate = value['plate']
+
+                # red box
+                cv2.rectangle(im, (int(x1), int(y1)),
+                              (int(x1 + w), int(y1 + h)), (0, 0, 255), 2)
+
+                cv2.putText(
+                    im,
+                    "illegal_parking:" + plate,
+                    (int(x1) + 5, int(16 * text_scale + y1 + 15)),
+                    cv2.FONT_ITALIC,
+                    text_scale * 1.5, (0, 0, 255),
+                    thickness=text_thickness)
 
     for cls_id in range(num_classes):
         tlwhs = tlwhs_dict[cls_id]