From 0b463621cc747c2bdc1bf47a4565a7a7960cdb87 Mon Sep 17 00:00:00 2001 From: XYZ <1290573099@qq.com> Date: Mon, 15 Aug 2022 16:23:17 +0800 Subject: [PATCH] add md5sum check and illegal parking recognition (#6608) * 1. add md5sum check; 2. add illegal parking recognition. * add config for illegal parking recognition * car licenseplate skipframe&time counter vote (#6609) * car licenseplate skipframe&time counter vote * incase of no track * update the method to get plate * if illegal_parking_time is -1, do not update_object_info * get model dir from cfg * illegal parking visualize optimize Co-authored-by: zhiboniu <31800336+zhiboniu@users.noreply.github.com> --- deploy/pipeline/cfg_utils.py | 6 ++ .../examples/infer_cfg_illegal_parking.yml | 19 ++++ deploy/pipeline/download.py | 31 ++++++- deploy/pipeline/pipeline.py | 42 +++++++-- deploy/pptracking/python/mot/utils.py | 91 ++++++++++++++++++- deploy/pptracking/python/mot/visualize.py | 42 +++++++-- 6 files changed, 212 insertions(+), 19 deletions(-) create mode 100644 deploy/pipeline/config/examples/infer_cfg_illegal_parking.yml diff --git a/deploy/pipeline/cfg_utils.py b/deploy/pipeline/cfg_utils.py index 1e42b29c0..0999b8bb3 100644 --- a/deploy/pipeline/cfg_utils.py +++ b/deploy/pipeline/cfg_utils.py @@ -127,6 +127,12 @@ def argsparser(): help="Whether counting the numbers of identifiers break in " "the area. Note that only support single-class MOT and " "the video should be taken by a static camera.") + parser.add_argument( + "--illegal_parking_time", + type=int, + default=-1, + help="illegal parking time which units are seconds, default is -1 which means not recognition illegal parking" + ) parser.add_argument( "--region_type", type=str, diff --git a/deploy/pipeline/config/examples/infer_cfg_illegal_parking.yml b/deploy/pipeline/config/examples/infer_cfg_illegal_parking.yml new file mode 100644 index 000000000..43030a6af --- /dev/null +++ b/deploy/pipeline/config/examples/infer_cfg_illegal_parking.yml @@ -0,0 +1,19 @@ +crop_thresh: 0.5 +visual: True +warmup_frame: 50 + +MOT: + model_dir: https://bj.bcebos.com/v1/paddledet/models/pipeline/mot_ppyoloe_l_36e_ppvehicle.zip + tracker_config: deploy/pipeline/config/tracker_config.yml + batch_size: 1 + enable: True + +VEHICLE_PLATE: + det_model_dir: https://paddleocr.bj.bcebos.com/PP-OCRv3/chinese/ch_PP-OCRv3_det_infer.tar + det_limit_side_len: 480 + det_limit_type: "max" + rec_model_dir: https://paddleocr.bj.bcebos.com/PP-OCRv3/chinese/ch_PP-OCRv3_rec_infer.tar + rec_image_shape: [3, 48, 320] + rec_batch_num: 6 + word_dict_path: deploy/pipeline/ppvehicle/rec_word_dict.txt + enable: True diff --git a/deploy/pipeline/download.py b/deploy/pipeline/download.py index f243838b7..fd9060a97 100644 --- a/deploy/pipeline/download.py +++ b/deploy/pipeline/download.py @@ -29,6 +29,31 @@ DOWNLOAD_RETRY_LIMIT = 3 WEIGHTS_HOME = osp.expanduser("~/.cache/paddle/infer_weights") +MODEL_URL_MD5_DICT = { + "https://paddleocr.bj.bcebos.com/PP-OCRv3/chinese/ch_PP-OCRv3_det_infer.tar": + '982d2d8d83e55f5f981e96a7b941fff5', + "https://paddleocr.bj.bcebos.com/PP-OCRv3/chinese/ch_PP-OCRv3_rec_infer.tar": + '5f021b88518bdeda2cb4a3aacc481024', + "https://bj.bcebos.com/v1/paddledet/models/pipeline/mot_ppyoloe_l_36e_ppvehicle.zip": + "3859d1a26e0c498285c2374b1a347013", + "https://bj.bcebos.com/v1/paddledet/models/pipeline/mot_ppyoloe_l_36e_ppvehicle.zip": + "46a80e1b3a8f4599e0cc79367c195b7c", + "https://bj.bcebos.com/v1/paddledet/models/pipeline/dark_hrnet_w32_256x192.zip": + "a20d5f6ca087bff0e9f2b18df45a36f2", + "https://bj.bcebos.com/v1/paddledet/models/pipeline/PPLCNet_x1_0_person_attribute_945_infer.zip": + "1dfb161bf12bbc1365b2ed6866674483", + "https://videotag.bj.bcebos.com/PaddleVideo-release2.3/ppTSM_fight.zip": + "5d4609142501258608bf0a1445eedaba", + "https://bj.bcebos.com/v1/paddledet/models/pipeline/STGCN.zip": + "cf1c3c4bae90b975accb954d13129ea4", + "https://bj.bcebos.com/v1/paddledet/models/pipeline/ppyoloe_crn_s_80e_smoking_visdrone.zip": + "4cd12ae55be8f0eb2b90c08ac3b48218", + "https://bj.bcebos.com/v1/paddledet/models/pipeline/PPHGNet_tiny_calling_halfbody.zip": + "cf86b87ace97540dace6ef08e62b584a", + "https://bj.bcebos.com/v1/paddledet/models/pipeline/reid_model.zip": + "fdc4dac38393b8e2b5921c1e1fdd5315" +} + def is_url(path): """ @@ -91,7 +116,6 @@ def _download(url, path, md5sum=None): fname = osp.split(url)[-1] fullname = osp.join(path, fname) retry_cnt = 0 - while not (osp.exists(fullname) and _check_exist_file_md5(fullname, md5sum, url)): if retry_cnt < DOWNLOAD_RETRY_LIMIT: @@ -296,7 +320,10 @@ def get_weights_path(url): download it from url. """ url = parse_url(url) - path, _ = get_path(url, WEIGHTS_HOME) + md5sum = None + if url in MODEL_URL_MD5_DICT.keys(): + md5sum = MODEL_URL_MD5_DICT[url] + path, _ = get_path(url, WEIGHTS_HOME, md5sum) return path diff --git a/deploy/pipeline/pipeline.py b/deploy/pipeline/pipeline.py index 6acd808b1..1190d22d5 100644 --- a/deploy/pipeline/pipeline.py +++ b/deploy/pipeline/pipeline.py @@ -40,7 +40,7 @@ from python.visualize import visualize_box_mask, visualize_attr, visualize_pose, from pptracking.python.mot_sde_infer import SDE_Detector from pptracking.python.mot.visualize import plot_tracking_dict -from pptracking.python.mot.utils import flow_statistic +from pptracking.python.mot.utils import flow_statistic, update_object_info from pphuman.attr_infer import AttrDetector from pphuman.video_action_infer import VideoActionRecognizer @@ -283,6 +283,7 @@ class PipePredictor(object): self.is_video = is_video self.multi_camera = multi_camera self.cfg = cfg + self.output_dir = args.output_dir self.draw_center_traj = args.draw_center_traj self.secs_interval = args.secs_interval @@ -290,6 +291,7 @@ class PipePredictor(object): self.do_break_in_counting = args.do_break_in_counting self.region_type = args.region_type self.region_polygon = args.region_polygon + self.illegal_parking_time = args.illegal_parking_time self.warmup_frame = self.cfg['warmup_frame'] self.pipeline_res = Result() @@ -376,6 +378,13 @@ class PipePredictor(object): use_dark=False) self.kpt_buff = KeyPointBuff(skeleton_action_frames) + if self.with_vehicleplate: + vehicleplate_cfg = self.cfg['VEHICLE_PLATE'] + self.vehicleplate_detector = PlateRecognizer(args, + vehicleplate_cfg) + basemode = self.basemode['VEHICLE_PLATE'] + self.modebase[basemode] = True + if self.with_mtmct: reid_cfg = self.cfg['REID'] basemode = self.basemode['REID'] @@ -545,7 +554,7 @@ class PipePredictor(object): out_id_list = list() prev_center = dict() records = list() - if self.do_entrance_counting or self.do_break_in_counting: + if self.do_entrance_counting or self.do_break_in_counting or self.illegal_parking_time != -1: if self.region_type == 'horizontal': entrance = [0, height / 2., width, height / 2.] elif self.region_type == 'vertical': @@ -575,6 +584,10 @@ class PipePredictor(object): short_size = self.cfg["VIDEO_ACTION"]["short_size"] scale = ShortSizeScale(short_size) + object_in_region_info = { + } # store info for vehicle parking in region + illegal_parking_dict = None + while (1): if frame_id % 10 == 0: print('frame id: ', frame_id) @@ -608,6 +621,16 @@ class PipePredictor(object): prev_center, records) records = statistic['records'] + if self.illegal_parking_time != -1: + object_in_region_info, illegal_parking_dict = update_object_info( + object_in_region_info, mot_result, self.region_type, + entrance, video_fps, self.illegal_parking_time) + if len(illegal_parking_dict) != 0: + # build relationship between id and plate + for key, value in illegal_parking_dict.items(): + plate = self.collector.get_carlp(key) + illegal_parking_dict[key]['plate'] = plate + # nothing detected if len(mot_res['boxes']) == 0: frame_id += 1 @@ -786,9 +809,12 @@ class PipePredictor(object): if self.cfg['visual']: _, _, fps = self.pipe_timer.get_total_time() - im = self.visualize_video( - frame, self.pipeline_res, self.collector, frame_id, fps, - entrance, records, center_traj) # visualize + + im = self.visualize_video(frame, self.pipeline_res, + self.collector, frame_id, fps, + entrance, records, center_traj, + self.illegal_parking_time != -1, + illegal_parking_dict) # visualize writer.write(im) if self.file_name is None: # use camera_id cv2.imshow('Paddle-Pipeline', im) @@ -806,7 +832,9 @@ class PipePredictor(object): fps, entrance=None, records=None, - center_traj=None): + center_traj=None, + do_illegal_parking_recognition=False, + illegal_parking_dict=None): mot_res = copy.deepcopy(result.get('mot')) if mot_res is not None: ids = mot_res['boxes'][:, 0] @@ -840,6 +868,8 @@ class PipePredictor(object): ids2names=self.mot_predictor.pred_config.labels, do_entrance_counting=self.do_entrance_counting, do_break_in_counting=self.do_break_in_counting, + do_illegal_parking_recognition=do_illegal_parking_recognition, + illegal_parking_dict=illegal_parking_dict, entrance=entrance, records=records, center_traj=center_traj) diff --git a/deploy/pptracking/python/mot/utils.py b/deploy/pptracking/python/mot/utils.py index 503589d81..d31fe7d17 100644 --- a/deploy/pptracking/python/mot/utils.py +++ b/deploy/pptracking/python/mot/utils.py @@ -17,10 +17,12 @@ import cv2 import time import numpy as np import collections +import math __all__ = [ 'MOTTimer', 'Detection', 'write_mot_results', 'load_det_results', - 'preprocess_reid', 'get_crops', 'clip_box', 'scale_coords', 'flow_statistic' + 'preprocess_reid', 'get_crops', 'clip_box', 'scale_coords', + 'flow_statistic', 'update_object_info' ] @@ -335,6 +337,93 @@ def flow_statistic(result, } +def distance(center_1, center_2): + return math.sqrt( + math.pow(center_1[0] - center_2[0], 2) + math.pow(center_1[1] - + center_2[1], 2)) + + +# update vehicle parking info +def update_object_info(object_in_region_info, + result, + region_type, + entrance, + fps, + illegal_parking_time, + distance_threshold_frame=3, + distance_threshold_interval=50): + ''' + For consecutive frames, the distance between two frame is smaller than distance_threshold_frame, regard as parking + For parking in general, the move distance should smaller than distance_threshold_interval + The moving distance of the vehicle is scaled according to the y, which is inversely proportional to y. + ''' + + assert region_type in [ + 'custom' + ], "region_type should be 'custom' when do break_in counting." + assert len( + entrance + ) >= 4, "entrance should be at least 3 points and (w,h) of image when do break_in counting." + + frame_id, tlwhs, tscores, track_ids = result # result from mot + + im_w, im_h = entrance[-1][:] + entrance = np.array(entrance[:-1]) + + illegal_parking_dict = {} + for tlwh, score, track_id in zip(tlwhs, tscores, track_ids): + if track_id < 0: continue + + x1, y1, w, h = tlwh + center_x = min(x1 + w / 2., im_w - 1) + center_y = min(y1 + h / 2, im_h - 1) + + if not in_quadrangle([center_x, center_y], entrance, im_h, im_w): + continue + + current_center = (center_x, center_y) + if track_id not in object_in_region_info.keys( + ): # first time appear in region + object_in_region_info[track_id] = {} + object_in_region_info[track_id]["start_frame"] = frame_id + object_in_region_info[track_id]["end_frame"] = frame_id + object_in_region_info[track_id]["prev_center"] = current_center + object_in_region_info[track_id]["start_center"] = current_center + else: + prev_center = object_in_region_info[track_id]["prev_center"] + + dis = distance(current_center, prev_center) + scaled_dis = 200 * dis / ( + current_center[1] + 1) # scale distance according to y + dis = scaled_dis + + if dis < distance_threshold_frame: # not move + object_in_region_info[track_id]["end_frame"] = frame_id + object_in_region_info[track_id]["prev_center"] = current_center + else: # move + object_in_region_info[track_id]["start_frame"] = frame_id + object_in_region_info[track_id]["end_frame"] = frame_id + object_in_region_info[track_id]["prev_center"] = current_center + object_in_region_info[track_id]["start_center"] = current_center + + # whether current object parking + distance_from_start = distance( + object_in_region_info[track_id]["start_center"], current_center) + if distance_from_start > distance_threshold_interval: + # moved + object_in_region_info[track_id]["start_frame"] = frame_id + object_in_region_info[track_id]["end_frame"] = frame_id + object_in_region_info[track_id]["prev_center"] = current_center + object_in_region_info[track_id]["start_center"] = current_center + continue + + if (object_in_region_info[track_id]["end_frame"]-object_in_region_info[track_id]["start_frame"]) /fps >= illegal_parking_time \ + and distance_from_start