pipeline.py 54.7 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import yaml
import glob
import cv2
import numpy as np
import math
import paddle
import sys
Z
zhiboniu 已提交
23
import copy
24 25 26
import threading
import queue
import time
Z
zhiboniu 已提交
27
from collections import Sequence, defaultdict
Z
zhiboniu 已提交
28
from datacollector import DataCollector, Result
29 30 31 32 33

# add deploy path of PadleDetection to sys.path
parent_path = os.path.abspath(os.path.join(__file__, *(['..'] * 2)))
sys.path.insert(0, parent_path)

34 35
from cfg_utils import argsparser, print_arguments, merge_cfg
from pipe_utils import PipeTimer
Z
zhiboniu 已提交
36
from pipe_utils import get_test_images, crop_image_with_det, crop_image_with_mot, parse_mot_res, parse_mot_keypoint
Z
zhiboniu 已提交
37
from pipe_utils import PushStream
Z
zhiboniu 已提交
38

39
from python.infer import Detector, DetectorPicoDet
J
JYChen 已提交
40 41
from python.keypoint_infer import KeyPointDetector
from python.keypoint_postprocess import translate_to_ori_images
42
from python.preprocess import decode_image, ShortSizeScale
L
LokeZhou 已提交
43
from python.visualize import visualize_box_mask, visualize_attr, visualize_pose, visualize_action, visualize_vehicleplate, visualize_vehiclepress, visualize_lane, visualize_vehicle_retrograde
44 45

from pptracking.python.mot_sde_infer import SDE_Detector
46
from pptracking.python.mot.visualize import plot_tracking_dict
47
from pptracking.python.mot.utils import flow_statistic, update_object_info
48

Z
zhiboniu 已提交
49 50 51 52 53 54 55
from pphuman.attr_infer import AttrDetector
from pphuman.video_action_infer import VideoActionRecognizer
from pphuman.action_infer import SkeletonActionRecognizer, DetActionRecognizer, ClsActionRecognizer
from pphuman.action_utils import KeyPointBuff, ActionVisualHelper
from pphuman.reid import ReID
from pphuman.mtmct import mtmct_process

56 57
from ppvehicle.vehicle_plate import PlateRecognizer
from ppvehicle.vehicle_attr import VehicleAttr
L
LokeZhou 已提交
58 59 60
from ppvehicle.vehicle_pressing import VehiclePressingRecognizer
from ppvehicle.vehicle_retrograde import VehicleRetrogradeRecognizer
from ppvehicle.lane_seg_infer import LaneSegPredictor
61

62 63
from download import auto_download_model

64 65 66 67 68 69

class Pipeline(object):
    """
    Pipeline

    Args:
J
JYChen 已提交
70
        args (argparse.Namespace): arguments in pipeline, which contains environment and runtime settings
71 72 73
        cfg (dict): config of models in pipeline
    """

Z
zhiboniu 已提交
74
    def __init__(self, args, cfg):
75
        self.multi_camera = False
Z
zhiboniu 已提交
76 77
        reid_cfg = cfg.get('REID', False)
        self.enable_mtmct = reid_cfg['enable'] if reid_cfg else False
78
        self.is_video = False
Z
zhiboniu 已提交
79
        self.output_dir = args.output_dir
Z
zhiboniu 已提交
80
        self.vis_result = cfg['visual']
Z
zhiboniu 已提交
81 82
        self.input = self._parse_input(args.image_file, args.image_dir,
                                       args.video_file, args.video_dir,
83
                                       args.camera_id, args.rtsp)
84
        if self.multi_camera:
85 86 87
            self.predictor = []
            for name in self.input:
                predictor_item = PipePredictor(
Z
zhiboniu 已提交
88
                    args, cfg, is_video=True, multi_camera=True)
89 90 91
                predictor_item.set_file_name(name)
                self.predictor.append(predictor_item)

92
        else:
Z
zhiboniu 已提交
93
            self.predictor = PipePredictor(args, cfg, self.is_video)
94
            if self.is_video:
95
                self.predictor.set_file_name(self.input)
96

Z
zhiboniu 已提交
97
    def _parse_input(self, image_file, image_dir, video_file, video_dir,
98
                     camera_id, rtsp):
99 100 101 102 103 104 105 106 107

        # parse input as is_video and multi_camera

        if image_file is not None or image_dir is not None:
            input = get_test_images(image_dir, image_file)
            self.is_video = False
            self.multi_camera = False

        elif video_file is not None:
Z
zhiboniu 已提交
108 109 110
            assert os.path.exists(
                video_file
            ) or 'rtsp' in video_file, "video_file not exists and not an rtsp site."
Z
zhiboniu 已提交
111 112 113 114 115 116 117
            self.multi_camera = False
            input = video_file
            self.is_video = True

        elif video_dir is not None:
            videof = [os.path.join(video_dir, x) for x in os.listdir(video_dir)]
            if len(videof) > 1:
118
                self.multi_camera = True
Z
zhiboniu 已提交
119 120
                videof.sort()
                input = videof
121
            else:
Z
zhiboniu 已提交
122
                input = videof[0]
123 124
            self.is_video = True

125 126 127 128 129 130 131 132 133 134
        elif rtsp is not None:
            if len(rtsp) > 1:
                rtsp = [rtsp_item for rtsp_item in rtsp if 'rtsp' in rtsp_item]
                self.multi_camera = True
                input = rtsp
            else:
                self.multi_camera = False
                input = rtsp[0]
            self.is_video = True

135
        elif camera_id != -1:
Z
zhiboniu 已提交
136 137
            self.multi_camera = False
            input = camera_id
138 139 140 141
            self.is_video = True

        else:
            raise ValueError(
142
                "Illegal Input, please set one of ['video_file', 'camera_id', 'image_file', 'image_dir']"
143 144 145 146
            )

        return input

147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176
    def run_multithreads(self):
        if self.multi_camera:
            multi_res = []
            threads = []
            for idx, (predictor,
                      input) in enumerate(zip(self.predictor, self.input)):
                thread = threading.Thread(
                    name=str(idx).zfill(3),
                    target=predictor.run,
                    args=(input, idx))
                threads.append(thread)

            for thread in threads:
                thread.start()

            for predictor, thread in zip(self.predictor, threads):
                thread.join()
                collector_data = predictor.get_result()
                multi_res.append(collector_data)

            if self.enable_mtmct:
                mtmct_process(
                    multi_res,
                    self.input,
                    mtmct_vis=self.vis_result,
                    output_dir=self.output_dir)

        else:
            self.predictor.run(self.input)

177 178 179 180 181
    def run(self):
        if self.multi_camera:
            multi_res = []
            for predictor, input in zip(self.predictor, self.input):
                predictor.run(input)
Z
zhiboniu 已提交
182 183
                collector_data = predictor.get_result()
                multi_res.append(collector_data)
184 185 186 187 188 189
            if self.enable_mtmct:
                mtmct_process(
                    multi_res,
                    self.input,
                    mtmct_vis=self.vis_result,
                    output_dir=self.output_dir)
190 191 192 193 194

        else:
            self.predictor.run(self.input)


195
def get_model_dir(cfg):
J
JYChen 已提交
196 197 198 199
    """ 
        Auto download inference model if the model_path is a url link. 
        Otherwise it will use the model_path directly.
    """
200 201 202 203 204 205 206 207 208 209
    for key in cfg.keys():
        if type(cfg[key]) ==  dict and \
            ("enable" in cfg[key].keys() and cfg[key]['enable']
                or "enable" not in cfg[key].keys()):

            if "model_dir" in cfg[key].keys():
                model_dir = cfg[key]["model_dir"]
                downloaded_model_dir = auto_download_model(model_dir)
                if downloaded_model_dir:
                    model_dir = downloaded_model_dir
J
JYChen 已提交
210 211
                    cfg[key]["model_dir"] = model_dir
                print(key, " model dir: ", model_dir)
212 213 214 215 216
            elif key == "VEHICLE_PLATE":
                det_model_dir = cfg[key]["det_model_dir"]
                downloaded_det_model_dir = auto_download_model(det_model_dir)
                if downloaded_det_model_dir:
                    det_model_dir = downloaded_det_model_dir
J
JYChen 已提交
217 218
                    cfg[key]["det_model_dir"] = det_model_dir
                print("det_model_dir model dir: ", det_model_dir)
219 220 221 222 223

                rec_model_dir = cfg[key]["rec_model_dir"]
                downloaded_rec_model_dir = auto_download_model(rec_model_dir)
                if downloaded_rec_model_dir:
                    rec_model_dir = downloaded_rec_model_dir
J
JYChen 已提交
224 225 226
                    cfg[key]["rec_model_dir"] = rec_model_dir
                print("rec_model_dir model dir: ", rec_model_dir)

227 228 229 230 231
        elif key == "MOT":  # for idbased and skeletonbased actions
            model_dir = cfg[key]["model_dir"]
            downloaded_model_dir = auto_download_model(model_dir)
            if downloaded_model_dir:
                model_dir = downloaded_model_dir
J
JYChen 已提交
232 233
                cfg[key]["model_dir"] = model_dir
            print("mot_model_dir model_dir: ", model_dir)
234 235


236 237 238 239 240 241 242 243 244 245 246 247 248
class PipePredictor(object):
    """
    Predictor in single camera
    
    The pipeline for image input: 

        1. Detection
        2. Detection -> Attribute

    The pipeline for video input: 

        1. Tracking
        2. Tracking -> Attribute
Z
zhiboniu 已提交
249
        3. Tracking -> KeyPoint -> SkeletonAction Recognition
250
        4. VideoAction Recognition
251 252

    Args:
J
JYChen 已提交
253
        args (argparse.Namespace): arguments in pipeline, which contains environment and runtime settings
254 255 256 257 258 259
        cfg (dict): config of models in pipeline
        is_video (bool): whether the input is video, default as False
        multi_camera (bool): whether to use multi camera in pipeline, 
            default as False
    """

Z
zhiboniu 已提交
260 261 262 263
    def __init__(self, args, cfg, is_video=True, multi_camera=False):
        # general module for pphuman and ppvehicle
        self.with_mot = cfg.get('MOT', False)['enable'] if cfg.get(
            'MOT', False) else False
264
        self.with_human_attr = cfg.get('ATTR', False)['enable'] if cfg.get(
Z
zhiboniu 已提交
265
            'ATTR', False) else False
Z
zhiboniu 已提交
266 267
        if self.with_mot:
            print('Multi-Object Tracking enabled')
268 269
        if self.with_human_attr:
            print('Human Attribute Recognition enabled')
Z
zhiboniu 已提交
270 271

        # only for pphuman
Z
zhiboniu 已提交
272 273 274
        self.with_skeleton_action = cfg.get(
            'SKELETON_ACTION', False)['enable'] if cfg.get('SKELETON_ACTION',
                                                           False) else False
Z
zhiboniu 已提交
275 276 277 278 279 280 281 282 283
        self.with_video_action = cfg.get(
            'VIDEO_ACTION', False)['enable'] if cfg.get('VIDEO_ACTION',
                                                        False) else False
        self.with_idbased_detaction = cfg.get(
            'ID_BASED_DETACTION', False)['enable'] if cfg.get(
                'ID_BASED_DETACTION', False) else False
        self.with_idbased_clsaction = cfg.get(
            'ID_BASED_CLSACTION', False)['enable'] if cfg.get(
                'ID_BASED_CLSACTION', False) else False
Z
zhiboniu 已提交
284 285
        self.with_mtmct = cfg.get('REID', False)['enable'] if cfg.get(
            'REID', False) else False
286

Z
zhiboniu 已提交
287 288
        if self.with_skeleton_action:
            print('SkeletonAction Recognition enabled')
Z
zhiboniu 已提交
289 290 291 292 293 294
        if self.with_video_action:
            print('VideoAction Recognition enabled')
        if self.with_idbased_detaction:
            print('IDBASED Detection Action Recognition enabled')
        if self.with_idbased_clsaction:
            print('IDBASED Classification Action Recognition enabled')
Z
zhiboniu 已提交
295 296
        if self.with_mtmct:
            print("MTMCT enabled")
W
wangguanzhong 已提交
297

Z
zhiboniu 已提交
298 299 300 301 302 303 304
        # only for ppvehicle
        self.with_vehicleplate = cfg.get(
            'VEHICLE_PLATE', False)['enable'] if cfg.get('VEHICLE_PLATE',
                                                         False) else False
        if self.with_vehicleplate:
            print('Vehicle Plate Recognition enabled')

305 306 307 308 309 310
        self.with_vehicle_attr = cfg.get(
            'VEHICLE_ATTR', False)['enable'] if cfg.get('VEHICLE_ATTR',
                                                        False) else False
        if self.with_vehicle_attr:
            print('Vehicle Attribute Recognition enabled')

L
LokeZhou 已提交
311 312 313 314 315 316 317 318 319 320 321 322
        self.with_vehicle_press = cfg.get(
            'VEHICLE_PRESSING', False)['enable'] if cfg.get('VEHICLE_PRESSING',
                                                            False) else False
        if self.with_vehicle_press:
            print('Vehicle Pressing Recognition enabled')

        self.with_vehicle_retrograde = cfg.get(
            'VEHICLE_RETROGRADE', False)['enable'] if cfg.get(
                'VEHICLE_RETROGRADE', False) else False
        if self.with_vehicle_retrograde:
            print('Vehicle Retrograde Recognition enabled')

323 324 325 326 327 328
        self.modebase = {
            "framebased": False,
            "videobased": False,
            "idbased": False,
            "skeletonbased": False
        }
329

330 331 332 333 334 335 336 337 338 339
        self.basemode = {
            "MOT": "idbased",
            "ATTR": "idbased",
            "VIDEO_ACTION": "videobased",
            "SKELETON_ACTION": "skeletonbased",
            "ID_BASED_DETACTION": "idbased",
            "ID_BASED_CLSACTION": "idbased",
            "REID": "idbased",
            "VEHICLE_PLATE": "idbased",
            "VEHICLE_ATTR": "idbased",
L
LokeZhou 已提交
340 341
            "VEHICLE_PRESSING": "idbased",
            "VEHICLE_RETROGRADE": "idbased",
342 343
        }

344 345 346
        self.is_video = is_video
        self.multi_camera = multi_camera
        self.cfg = cfg
347

J
JYChen 已提交
348 349 350 351 352 353 354
        self.output_dir = args.output_dir
        self.draw_center_traj = args.draw_center_traj
        self.secs_interval = args.secs_interval
        self.do_entrance_counting = args.do_entrance_counting
        self.do_break_in_counting = args.do_break_in_counting
        self.region_type = args.region_type
        self.region_polygon = args.region_polygon
355
        self.illegal_parking_time = args.illegal_parking_time
356

J
JYChen 已提交
357
        self.warmup_frame = self.cfg['warmup_frame']
358 359
        self.pipeline_res = Result()
        self.pipe_timer = PipeTimer()
360
        self.file_name = None
Z
zhiboniu 已提交
361
        self.collector = DataCollector()
362

Z
zhiboniu 已提交
363 364
        self.pushurl = args.pushurl

365
        # auto download inference model
J
JYChen 已提交
366
        get_model_dir(self.cfg)
367

Z
zhiboniu 已提交
368 369 370 371 372 373 374 375 376 377
        if self.with_vehicleplate:
            vehicleplate_cfg = self.cfg['VEHICLE_PLATE']
            self.vehicleplate_detector = PlateRecognizer(args, vehicleplate_cfg)
            basemode = self.basemode['VEHICLE_PLATE']
            self.modebase[basemode] = True

        if self.with_human_attr:
            attr_cfg = self.cfg['ATTR']
            basemode = self.basemode['ATTR']
            self.modebase[basemode] = True
J
JYChen 已提交
378
            self.attr_predictor = AttrDetector.init_with_cfg(args, attr_cfg)
Z
zhiboniu 已提交
379 380 381 382 383

        if self.with_vehicle_attr:
            vehicleattr_cfg = self.cfg['VEHICLE_ATTR']
            basemode = self.basemode['VEHICLE_ATTR']
            self.modebase[basemode] = True
J
JYChen 已提交
384 385
            self.vehicle_attr_predictor = VehicleAttr.init_with_cfg(
                args, vehicleattr_cfg)
Z
zhiboniu 已提交
386

L
LokeZhou 已提交
387 388 389 390 391 392 393 394 395 396 397 398
        if self.with_vehicle_press:
            vehiclepress_cfg = self.cfg['VEHICLE_PRESSING']
            basemode = self.basemode['VEHICLE_PRESSING']
            self.modebase[basemode] = True
            self.vehicle_press_predictor = VehiclePressingRecognizer(
                vehiclepress_cfg)

        if self.with_vehicle_press or self.with_vehicle_retrograde:
            laneseg_cfg = self.cfg['LANE_SEG']
            self.laneseg_predictor = LaneSegPredictor(
                laneseg_cfg['lane_seg_config'], laneseg_cfg['model_dir'])

399
        if not is_video:
L
LokeZhou 已提交
400

401
            det_cfg = self.cfg['DET']
J
JYChen 已提交
402
            model_dir = det_cfg['model_dir']
403 404
            batch_size = det_cfg['batch_size']
            self.det_predictor = Detector(
J
JYChen 已提交
405 406 407
                model_dir, args.device, args.run_mode, batch_size,
                args.trt_min_shape, args.trt_max_shape, args.trt_opt_shape,
                args.trt_calib_mode, args.cpu_threads, args.enable_mkldnn)
408
        else:
Z
zhiboniu 已提交
409
            if self.with_idbased_detaction:
J
JYChen 已提交
410
                idbased_detaction_cfg = self.cfg['ID_BASED_DETACTION']
411
                basemode = self.basemode['ID_BASED_DETACTION']
J
JYChen 已提交
412
                self.modebase[basemode] = True
413

J
JYChen 已提交
414 415
                self.det_action_predictor = DetActionRecognizer.init_with_cfg(
                    args, idbased_detaction_cfg)
J
JYChen 已提交
416 417
                self.det_action_visual_helper = ActionVisualHelper(1)

Z
zhiboniu 已提交
418
            if self.with_idbased_clsaction:
J
JYChen 已提交
419
                idbased_clsaction_cfg = self.cfg['ID_BASED_CLSACTION']
420
                basemode = self.basemode['ID_BASED_CLSACTION']
J
JYChen 已提交
421
                self.modebase[basemode] = True
422

J
JYChen 已提交
423 424
                self.cls_action_predictor = ClsActionRecognizer.init_with_cfg(
                    args, idbased_clsaction_cfg)
J
JYChen 已提交
425 426
                self.cls_action_visual_helper = ActionVisualHelper(1)

Z
zhiboniu 已提交
427 428 429 430
            if self.with_skeleton_action:
                skeleton_action_cfg = self.cfg['SKELETON_ACTION']
                display_frames = skeleton_action_cfg['display_frames']
                self.coord_size = skeleton_action_cfg['coord_size']
431
                basemode = self.basemode['SKELETON_ACTION']
432
                self.modebase[basemode] = True
J
JYChen 已提交
433
                skeleton_action_frames = skeleton_action_cfg['max_frames']
434

J
JYChen 已提交
435 436
                self.skeleton_action_predictor = SkeletonActionRecognizer.init_with_cfg(
                    args, skeleton_action_cfg)
J
JYChen 已提交
437
                self.skeleton_action_visual_helper = ActionVisualHelper(
Z
zhiboniu 已提交
438
                    display_frames)
439

J
JYChen 已提交
440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455
                kpt_cfg = self.cfg['KPT']
                kpt_model_dir = kpt_cfg['model_dir']
                kpt_batch_size = kpt_cfg['batch_size']
                self.kpt_predictor = KeyPointDetector(
                    kpt_model_dir,
                    args.device,
                    args.run_mode,
                    kpt_batch_size,
                    args.trt_min_shape,
                    args.trt_max_shape,
                    args.trt_opt_shape,
                    args.trt_calib_mode,
                    args.cpu_threads,
                    args.enable_mkldnn,
                    use_dark=False)
                self.kpt_buff = KeyPointBuff(skeleton_action_frames)
Z
zhiboniu 已提交
456

457 458 459 460 461 462 463
            if self.with_vehicleplate:
                vehicleplate_cfg = self.cfg['VEHICLE_PLATE']
                self.vehicleplate_detector = PlateRecognizer(args,
                                                             vehicleplate_cfg)
                basemode = self.basemode['VEHICLE_PLATE']
                self.modebase[basemode] = True

Z
zhiboniu 已提交
464 465
            if self.with_mtmct:
                reid_cfg = self.cfg['REID']
466
                basemode = self.basemode['REID']
Z
zhiboniu 已提交
467
                self.modebase[basemode] = True
J
JYChen 已提交
468
                self.reid_predictor = ReID.init_with_cfg(args, reid_cfg)
Z
zhiboniu 已提交
469

L
LokeZhou 已提交
470 471 472 473 474 475 476
            if self.with_vehicle_retrograde:
                vehicleretrograde_cfg = self.cfg['VEHICLE_RETROGRADE']
                basemode = self.basemode['VEHICLE_RETROGRADE']
                self.modebase[basemode] = True
                self.vehicle_retrograde_predictor = VehicleRetrogradeRecognizer(
                    vehicleretrograde_cfg)

Z
zhiboniu 已提交
477 478 479
            if self.with_mot or self.modebase["idbased"] or self.modebase[
                    "skeletonbased"]:
                mot_cfg = self.cfg['MOT']
J
JYChen 已提交
480
                model_dir = mot_cfg['model_dir']
Z
zhiboniu 已提交
481 482
                tracker_config = mot_cfg['tracker_config']
                batch_size = mot_cfg['batch_size']
483
                skip_frame_num = mot_cfg.get('skip_frame_num', -1)
484
                basemode = self.basemode['MOT']
Z
zhiboniu 已提交
485 486 487 488
                self.modebase[basemode] = True
                self.mot_predictor = SDE_Detector(
                    model_dir,
                    tracker_config,
J
JYChen 已提交
489 490
                    args.device,
                    args.run_mode,
Z
zhiboniu 已提交
491
                    batch_size,
J
JYChen 已提交
492 493 494 495 496 497
                    args.trt_min_shape,
                    args.trt_max_shape,
                    args.trt_opt_shape,
                    args.trt_calib_mode,
                    args.cpu_threads,
                    args.enable_mkldnn,
498
                    skip_frame_num=skip_frame_num,
J
JYChen 已提交
499 500 501 502 503 504
                    draw_center_traj=self.draw_center_traj,
                    secs_interval=self.secs_interval,
                    do_entrance_counting=self.do_entrance_counting,
                    do_break_in_counting=self.do_break_in_counting,
                    region_type=self.region_type,
                    region_polygon=self.region_polygon)
Z
zhiboniu 已提交
505

506 507
            if self.with_video_action:
                video_action_cfg = self.cfg['VIDEO_ACTION']
508
                basemode = self.basemode['VIDEO_ACTION']
509
                self.modebase[basemode] = True
J
JYChen 已提交
510 511
                self.video_action_predictor = VideoActionRecognizer.init_with_cfg(
                    args, video_action_cfg)
512

513
    def set_file_name(self, path):
L
LokeZhou 已提交
514
        if type(path) == int:
Z
zhiboniu 已提交
515 516
            self.file_name = path
        elif path is not None:
517 518 519
            self.file_name = os.path.split(path)[-1]
            if "." in self.file_name:
                self.file_name = self.file_name.split(".")[-2]
W
wangguanzhong 已提交
520 521 522
        else:
            # use camera id
            self.file_name = None
523

524
    def get_result(self):
Z
zhiboniu 已提交
525
        return self.collector.get_res()
526

527
    def run(self, input, thread_idx=0):
528
        if self.is_video:
529
            self.predict_video(input, thread_idx=thread_idx)
530 531
        else:
            self.predict_image(input)
532
        self.pipe_timer.info()
533
        self.mot_predictor.det_times.tracking_info(average=True)
534 535 536 537 538 539

    def predict_image(self, input):
        # det
        # det -> attr
        batch_loop_cnt = math.ceil(
            float(len(input)) / self.det_predictor.batch_size)
L
LokeZhou 已提交
540
        self.warmup_frame = min(10, len(input) // 2) - 1
541 542 543 544 545 546 547 548 549 550 551 552
        for i in range(batch_loop_cnt):
            start_index = i * self.det_predictor.batch_size
            end_index = min((i + 1) * self.det_predictor.batch_size, len(input))
            batch_file = input[start_index:end_index]
            batch_input = [decode_image(f, {})[0] for f in batch_file]

            if i > self.warmup_frame:
                self.pipe_timer.total_time.start()
                self.pipe_timer.module_time['det'].start()
            # det output format: class, score, xmin, ymin, xmax, ymax
            det_res = self.det_predictor.predict_image(
                batch_input, visual=False)
553 554
            det_res = self.det_predictor.filter_box(det_res,
                                                    self.cfg['crop_thresh'])
555 556
            if i > self.warmup_frame:
                self.pipe_timer.module_time['det'].end()
Z
zhiboniu 已提交
557
                self.pipe_timer.track_num += len(det_res['boxes'])
558 559
            self.pipeline_res.update(det_res, 'det')

560
            if self.with_human_attr:
561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577
                crop_inputs = crop_image_with_det(batch_input, det_res)
                attr_res_list = []

                if i > self.warmup_frame:
                    self.pipe_timer.module_time['attr'].start()

                for crop_input in crop_inputs:
                    attr_res = self.attr_predictor.predict_image(
                        crop_input, visual=False)
                    attr_res_list.extend(attr_res['output'])

                if i > self.warmup_frame:
                    self.pipe_timer.module_time['attr'].end()

                attr_res = {'output': attr_res_list}
                self.pipeline_res.update(attr_res, 'attr')

578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595
            if self.with_vehicle_attr:
                crop_inputs = crop_image_with_det(batch_input, det_res)
                vehicle_attr_res_list = []

                if i > self.warmup_frame:
                    self.pipe_timer.module_time['vehicle_attr'].start()

                for crop_input in crop_inputs:
                    attr_res = self.vehicle_attr_predictor.predict_image(
                        crop_input, visual=False)
                    vehicle_attr_res_list.extend(attr_res['output'])

                if i > self.warmup_frame:
                    self.pipe_timer.module_time['vehicle_attr'].end()

                attr_res = {'output': vehicle_attr_res_list}
                self.pipeline_res.update(attr_res, 'vehicle_attr')

Z
zhiboniu 已提交
596 597 598 599 600 601 602 603 604 605 606 607 608 609
            if self.with_vehicleplate:
                if i > self.warmup_frame:
                    self.pipe_timer.module_time['vehicleplate'].start()
                crop_inputs = crop_image_with_det(batch_input, det_res)
                platelicenses = []
                for crop_input in crop_inputs:
                    platelicense = self.vehicleplate_detector.get_platelicense(
                        crop_input)
                    platelicenses.extend(platelicense['plate'])
                if i > self.warmup_frame:
                    self.pipe_timer.module_time['vehicleplate'].end()
                vehicleplate_res = {'vehicleplate': platelicenses}
                self.pipeline_res.update(vehicleplate_res, 'vehicleplate')

L
LokeZhou 已提交
610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627
            if self.with_vehicle_press:
                vehicle_press_res_list = []
                if i > self.warmup_frame:
                    self.pipe_timer.module_time['vehicle_press'].start()

                lanes, direction = self.laneseg_predictor.run(batch_input)
                if len(lanes) == 0:
                    print(" no lanes!")
                    continue

                lanes_res = {'output': lanes, 'direction': direction}
                self.pipeline_res.update(lanes_res, 'lanes')

                vehicle_press_res_list = self.vehicle_press_predictor.run(
                    lanes, det_res)
                vehiclepress_res = {'output': vehicle_press_res_list}
                self.pipeline_res.update(vehiclepress_res, 'vehicle_press')

628 629 630 631 632 633 634
            self.pipe_timer.img_num += len(batch_input)
            if i > self.warmup_frame:
                self.pipe_timer.total_time.end()

            if self.cfg['visual']:
                self.visualize_image(batch_file, batch_input, self.pipeline_res)

635 636
    def capturevideo(self, capture, queue):
        frame_id = 0
L
LokeZhou 已提交
637
        while (1):
638 639 640 641 642 643 644 645 646
            if queue.full():
                time.sleep(0.1)
            else:
                ret, frame = capture.read()
                if not ret:
                    return
                frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
                queue.put(frame_rgb)

647
    def predict_video(self, video_file, thread_idx=0):
648 649 650
        # mot
        # mot -> attr
        # mot -> pose -> action
Z
zhiboniu 已提交
651
        capture = cv2.VideoCapture(video_file)
652 653 654 655 656 657

        # Get Video info : resolution, fps, frame count
        width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH))
        height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT))
        fps = int(capture.get(cv2.CAP_PROP_FPS))
        frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT))
658
        print("video fps: %d, frame_count: %d" % (fps, frame_count))
659

Z
zhiboniu 已提交
660 661 662 663 664 665 666
        if len(self.pushurl) > 0:
            video_out_name = 'output' if self.file_name is None else self.file_name
            pushurl = os.path.join(self.pushurl, video_out_name)
            print("the result will push stream to url:{}".format(pushurl))
            pushstream = PushStream(pushurl)
            pushstream.initcmd(fps, width, height)
        elif self.cfg['visual']:
L
LokeZhou 已提交
667 668 669 670
            video_out_name = 'output' if (
                self.file_name is None or
                type(self.file_name) == int) else self.file_name
            if type(video_file) == str and "rtsp" in video_file:
Z
zhiboniu 已提交
671 672 673 674
                video_out_name = video_out_name + "_t" + str(thread_idx).zfill(
                    2) + "_rtsp"
            if not os.path.exists(self.output_dir):
                os.makedirs(self.output_dir)
L
LokeZhou 已提交
675
            out_path = os.path.join(self.output_dir, video_out_name + ".mp4")
Z
zhiboniu 已提交
676 677 678
            fourcc = cv2.VideoWriter_fourcc(* 'mp4v')
            writer = cv2.VideoWriter(out_path, fourcc, fps, (width, height))

679
        frame_id = 0
680 681 682 683 684 685 686 687 688 689

        entrance, records, center_traj = None, None, None
        if self.draw_center_traj:
            center_traj = [{}]
        id_set = set()
        interval_id_set = set()
        in_id_list = list()
        out_id_list = list()
        prev_center = dict()
        records = list()
690
        if self.do_entrance_counting or self.do_break_in_counting or self.illegal_parking_time != -1:
691 692 693 694 695 696 697 698 699
            if self.region_type == 'horizontal':
                entrance = [0, height / 2., width, height / 2.]
            elif self.region_type == 'vertical':
                entrance = [width / 2, 0., width / 2, height]
            elif self.region_type == 'custom':
                entrance = []
                assert len(
                    self.region_polygon
                ) % 2 == 0, "region_polygon should be pairs of coords points when do break_in counting."
J
JYChen 已提交
700 701 702 703
                assert len(
                    self.region_polygon
                ) > 6, 'region_type is custom, region_polygon should be at least 3 pairs of point coords.'

704 705 706 707 708 709 710 711
                for i in range(0, len(self.region_polygon), 2):
                    entrance.append(
                        [self.region_polygon[i], self.region_polygon[i + 1]])
                entrance.append([width, height])
            else:
                raise ValueError("region_type:{} unsupported.".format(
                    self.region_type))

712 713
        video_fps = fps

714 715
        video_action_imgs = []

716 717 718 719
        if self.with_video_action:
            short_size = self.cfg["VIDEO_ACTION"]["short_size"]
            scale = ShortSizeScale(short_size)

720 721 722
        object_in_region_info = {
        }  # store info for vehicle parking in region       
        illegal_parking_dict = None
L
LokeZhou 已提交
723 724
        cars_count = 0
        retrograde_traj_len = 0
725 726 727
        framequeue = queue.Queue(10)

        thread = threading.Thread(
L
LokeZhou 已提交
728
            target=self.capturevideo, args=(capture, framequeue))
729 730 731
        thread.start()
        time.sleep(1)

L
LokeZhou 已提交
732
        while (not framequeue.empty()):
733
            if frame_id % 10 == 0:
734
                print('Thread: {}; frame id: {}'.format(thread_idx, frame_id))
735

736
            frame_rgb = framequeue.get()
Z
zhiboniu 已提交
737 738
            if frame_id > self.warmup_frame:
                self.pipe_timer.total_time.start()
739

740
            if self.modebase["idbased"] or self.modebase["skeletonbased"]:
741
                if frame_id > self.warmup_frame:
742
                    self.pipe_timer.module_time['mot'].start()
743

744 745 746 747 748 749 750
                mot_skip_frame_num = self.mot_predictor.skip_frame_num
                reuse_det_result = False
                if mot_skip_frame_num > 1 and frame_id > 0 and frame_id % mot_skip_frame_num > 0:
                    reuse_det_result = True
                res = self.mot_predictor.predict_image(
                    [copy.deepcopy(frame_rgb)],
                    visual=False,
751 752
                    reuse_det_result=reuse_det_result,
                    frame_count=frame_id)
753 754 755

                # mot output format: id, class, score, xmin, ymin, xmax, ymax
                mot_res = parse_mot_res(res)
Z
zhiboniu 已提交
756 757 758
                if frame_id > self.warmup_frame:
                    self.pipe_timer.module_time['mot'].end()
                    self.pipe_timer.track_num += len(mot_res['boxes'])
759

760 761 762 763
                if frame_id % 10 == 0:
                    print("Thread: {}; trackid number: {}".format(
                        thread_idx, len(mot_res['boxes'])))

764 765 766 767 768
                # flow_statistic only support single class MOT
                boxes, scores, ids = res[0]  # batch size = 1 in MOT
                mot_result = (frame_id + 1, boxes[0], scores[0],
                              ids[0])  # single class
                statistic = flow_statistic(
F
Feng Ni 已提交
769 770 771 772 773 774 775 776 777 778 779 780 781 782
                    mot_result,
                    self.secs_interval,
                    self.do_entrance_counting,
                    self.do_break_in_counting,
                    self.region_type,
                    video_fps,
                    entrance,
                    id_set,
                    interval_id_set,
                    in_id_list,
                    out_id_list,
                    prev_center,
                    records,
                    ids2names=self.mot_predictor.pred_config.labels)
783 784
                records = statistic['records']

785 786 787 788 789 790 791 792 793 794
                if self.illegal_parking_time != -1:
                    object_in_region_info, illegal_parking_dict = update_object_info(
                        object_in_region_info, mot_result, self.region_type,
                        entrance, video_fps, self.illegal_parking_time)
                    if len(illegal_parking_dict) != 0:
                        # build relationship between id and plate
                        for key, value in illegal_parking_dict.items():
                            plate = self.collector.get_carlp(key)
                            illegal_parking_dict[key]['plate'] = plate

795 796 797
                # nothing detected
                if len(mot_res['boxes']) == 0:
                    frame_id += 1
J
JYChen 已提交
798
                    if frame_id > self.warmup_frame:
799 800 801 802
                        self.pipe_timer.img_num += 1
                        self.pipe_timer.total_time.end()
                    if self.cfg['visual']:
                        _, _, fps = self.pipe_timer.get_total_time()
L
LokeZhou 已提交
803 804
                        im = self.visualize_video(frame_rgb, mot_res, frame_id,
                                                  fps, entrance, records,
805
                                                  center_traj)  # visualize
L
LokeZhou 已提交
806
                        if len(self.pushurl) > 0:
Z
zhiboniu 已提交
807 808 809 810 811 812 813
                            pushstream.pipe.stdin.write(im.tobytes())
                        else:
                            writer.write(im)
                            if self.file_name is None:  # use camera_id
                                cv2.imshow('Paddle-Pipeline', im)
                                if cv2.waitKey(1) & 0xFF == ord('q'):
                                    break
814 815 816
                    continue

                self.pipeline_res.update(mot_res, 'mot')
J
JYChen 已提交
817
                crop_input, new_bboxes, ori_bboxes = crop_image_with_mot(
818
                    frame_rgb, mot_res)
819

820
                if self.with_vehicleplate and frame_id % 10 == 0:
Z
zhiboniu 已提交
821 822
                    if frame_id > self.warmup_frame:
                        self.pipe_timer.module_time['vehicleplate'].start()
Z
zhiboniu 已提交
823 824
                    plate_input, _, _ = crop_image_with_mot(
                        frame_rgb, mot_res, expand=False)
Z
zhiboniu 已提交
825
                    platelicense = self.vehicleplate_detector.get_platelicense(
Z
zhiboniu 已提交
826
                        plate_input)
Z
zhiboniu 已提交
827 828
                    if frame_id > self.warmup_frame:
                        self.pipe_timer.module_time['vehicleplate'].end()
Z
zhiboniu 已提交
829
                    self.pipeline_res.update(platelicense, 'vehicleplate')
830 831
                else:
                    self.pipeline_res.clear('vehicleplate')
Z
zhiboniu 已提交
832

833
                if self.with_human_attr:
J
JYChen 已提交
834
                    if frame_id > self.warmup_frame:
835 836 837 838 839 840 841
                        self.pipe_timer.module_time['attr'].start()
                    attr_res = self.attr_predictor.predict_image(
                        crop_input, visual=False)
                    if frame_id > self.warmup_frame:
                        self.pipe_timer.module_time['attr'].end()
                    self.pipeline_res.update(attr_res, 'attr')

842 843 844 845 846 847 848 849 850
                if self.with_vehicle_attr:
                    if frame_id > self.warmup_frame:
                        self.pipe_timer.module_time['vehicle_attr'].start()
                    attr_res = self.vehicle_attr_predictor.predict_image(
                        crop_input, visual=False)
                    if frame_id > self.warmup_frame:
                        self.pipe_timer.module_time['vehicle_attr'].end()
                    self.pipeline_res.update(attr_res, 'vehicle_attr')

L
LokeZhou 已提交
851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883
                if self.with_vehicle_press or self.with_vehicle_retrograde:
                    if frame_id == 0 or cars_count == 0 or cars_count > len(
                            mot_res['boxes']):

                        if frame_id > self.warmup_frame:
                            self.pipe_timer.module_time['lanes'].start()
                        lanes, directions = self.laneseg_predictor.run(
                            [copy.deepcopy(frame_rgb)])
                        lanes_res = {'output': lanes, 'directions': directions}
                        if frame_id > self.warmup_frame:
                            self.pipe_timer.module_time['lanes'].end()

                        if frame_id == 0 or (len(lanes) > 0 and frame_id > 0):
                            self.pipeline_res.update(lanes_res, 'lanes')

                        cars_count = len(mot_res['boxes'])

                if self.with_vehicle_press:
                    if frame_id > self.warmup_frame:
                        self.pipe_timer.module_time['vehicle_press'].start()
                    press_lane = copy.deepcopy(self.pipeline_res.get('lanes'))
                    if press_lane is None:
                        continue

                    vehicle_press_res_list = self.vehicle_press_predictor.mot_run(
                        press_lane, mot_res['boxes'])
                    vehiclepress_res = {'output': vehicle_press_res_list}

                    if frame_id > self.warmup_frame:
                        self.pipe_timer.module_time['vehicle_press'].end()

                    self.pipeline_res.update(vehiclepress_res, 'vehicle_press')

Z
zhiboniu 已提交
884
                if self.with_idbased_detaction:
J
JYChen 已提交
885 886 887 888 889 890 891 892 893 894
                    if frame_id > self.warmup_frame:
                        self.pipe_timer.module_time['det_action'].start()
                    det_action_res = self.det_action_predictor.predict(
                        crop_input, mot_res)
                    if frame_id > self.warmup_frame:
                        self.pipe_timer.module_time['det_action'].end()
                    self.pipeline_res.update(det_action_res, 'det_action')

                    if self.cfg['visual']:
                        self.det_action_visual_helper.update(det_action_res)
Z
zhiboniu 已提交
895 896

                if self.with_idbased_clsaction:
J
JYChen 已提交
897 898 899 900 901 902 903 904 905 906
                    if frame_id > self.warmup_frame:
                        self.pipe_timer.module_time['cls_action'].start()
                    cls_action_res = self.cls_action_predictor.predict_with_mot(
                        crop_input, mot_res)
                    if frame_id > self.warmup_frame:
                        self.pipe_timer.module_time['cls_action'].end()
                    self.pipeline_res.update(cls_action_res, 'cls_action')

                    if self.cfg['visual']:
                        self.cls_action_visual_helper.update(cls_action_res)
Z
zhiboniu 已提交
907

Z
zhiboniu 已提交
908
                if self.with_skeleton_action:
Z
zhiboniu 已提交
909 910 911 912 913 914 915 916 917 918 919 920 921
                    if frame_id > self.warmup_frame:
                        self.pipe_timer.module_time['kpt'].start()
                    kpt_pred = self.kpt_predictor.predict_image(
                        crop_input, visual=False)
                    keypoint_vector, score_vector = translate_to_ori_images(
                        kpt_pred, np.array(new_bboxes))
                    kpt_res = {}
                    kpt_res['keypoint'] = [
                        keypoint_vector.tolist(), score_vector.tolist()
                    ] if len(keypoint_vector) > 0 else [[], []]
                    kpt_res['bbox'] = ori_bboxes
                    if frame_id > self.warmup_frame:
                        self.pipe_timer.module_time['kpt'].end()
922

Z
zhiboniu 已提交
923
                    self.pipeline_res.update(kpt_res, 'kpt')
924

Z
zhiboniu 已提交
925
                    self.kpt_buff.update(kpt_res, mot_res)  # collect kpt output
926 927 928
                    state = self.kpt_buff.get_state(
                    )  # whether frame num is enough or lost tracker

Z
zhiboniu 已提交
929
                    skeleton_action_res = {}
930 931
                    if state:
                        if frame_id > self.warmup_frame:
Z
zhiboniu 已提交
932 933
                            self.pipe_timer.module_time[
                                'skeleton_action'].start()
934 935
                        collected_keypoint = self.kpt_buff.get_collected_keypoint(
                        )  # reoragnize kpt output with ID
Z
zhiboniu 已提交
936 937 938 939
                        skeleton_action_input = parse_mot_keypoint(
                            collected_keypoint, self.coord_size)
                        skeleton_action_res = self.skeleton_action_predictor.predict_skeleton_with_mot(
                            skeleton_action_input)
940
                        if frame_id > self.warmup_frame:
Z
zhiboniu 已提交
941 942 943
                            self.pipe_timer.module_time['skeleton_action'].end()
                        self.pipeline_res.update(skeleton_action_res,
                                                 'skeleton_action')
944 945

                    if self.cfg['visual']:
Z
zhiboniu 已提交
946 947
                        self.skeleton_action_visual_helper.update(
                            skeleton_action_res)
948 949 950

                if self.with_mtmct and frame_id % 10 == 0:
                    crop_input, img_qualities, rects = self.reid_predictor.crop_image_with_mot(
951
                        frame_rgb, mot_res)
952 953 954 955 956 957
                    if frame_id > self.warmup_frame:
                        self.pipe_timer.module_time['reid'].start()
                    reid_res = self.reid_predictor.predict_batch(crop_input)

                    if frame_id > self.warmup_frame:
                        self.pipe_timer.module_time['reid'].end()
J
JYChen 已提交
958

959 960 961 962 963 964 965 966
                    reid_res_dict = {
                        'features': reid_res,
                        "qualities": img_qualities,
                        "rects": rects
                    }
                    self.pipeline_res.update(reid_res_dict, 'reid')
                else:
                    self.pipeline_res.clear('reid')
Z
zhiboniu 已提交
967

Z
zhiboniu 已提交
968
            if self.with_video_action:
969 970 971 972 973 974 975 976 977 978 979 980 981
                # get the params
                frame_len = self.cfg["VIDEO_ACTION"]["frame_len"]
                sample_freq = self.cfg["VIDEO_ACTION"]["sample_freq"]

                if sample_freq * frame_len > frame_count:  # video is too short
                    sample_freq = int(frame_count / frame_len)

                # filter the warmup frames
                if frame_id > self.warmup_frame:
                    self.pipe_timer.module_time['video_action'].start()

                # collect frames
                if frame_id % sample_freq == 0:
982
                    # Scale image
983
                    scaled_img = scale(frame_rgb)
984
                    video_action_imgs.append(scaled_img)
985 986 987 988 989 990 991 992 993 994 995 996 997 998

                # the number of collected frames is enough to predict video action
                if len(video_action_imgs) == frame_len:
                    classes, scores = self.video_action_predictor.predict(
                        video_action_imgs)
                    if frame_id > self.warmup_frame:
                        self.pipe_timer.module_time['video_action'].end()

                    video_action_res = {"class": classes[0], "score": scores[0]}
                    self.pipeline_res.update(video_action_res, 'video_action')

                    print("video_action_res:", video_action_res)

                    video_action_imgs.clear()  # next clip
Z
zhiboniu 已提交
999

L
LokeZhou 已提交
1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062
            if self.with_vehicle_retrograde:
                # get the params
                frame_len = self.cfg["VEHICLE_RETROGRADE"]["frame_len"]
                sample_freq = self.cfg["VEHICLE_RETROGRADE"]["sample_freq"]

                if sample_freq * frame_len > frame_count:  # video is too short
                    sample_freq = int(frame_count / frame_len)

                # filter the warmup frames
                if frame_id > self.warmup_frame:
                    self.pipe_timer.module_time['vehicle_retrograde'].start()

                if frame_id % sample_freq == 0:

                    frame_mot_res = copy.deepcopy(self.pipeline_res.get('mot'))
                    self.vehicle_retrograde_predictor.update_center_traj(
                        frame_mot_res, max_len=frame_len)
                    retrograde_traj_len = retrograde_traj_len + 1

                #the number of collected frames is enough to predict 
                if retrograde_traj_len == frame_len:
                    retrograde_mot_res = copy.deepcopy(
                        self.pipeline_res.get('mot'))
                    retrograde_lanes = copy.deepcopy(
                        self.pipeline_res.get('lanes'))
                    frame_shape = frame_rgb.shape

                    if retrograde_lanes is None:
                        continue
                    retrograde_res, fence_line = self.vehicle_retrograde_predictor.mot_run(
                        lanes_res=retrograde_lanes,
                        det_res=retrograde_mot_res,
                        frame_shape=frame_shape)

                    retrograde_res_update = self.pipeline_res.get(
                        'vehicle_retrograde')

                    if retrograde_res_update is not None:
                        retrograde_res_update = retrograde_res_update['output']
                        if retrograde_res is not None:
                            for retrograde_res_id in retrograde_res:
                                if retrograde_res_id not in retrograde_res_update:
                                    retrograde_res_update.append(
                                        retrograde_res_id)
                    else:
                        retrograde_res_update = []

                    retrograde_res_dict = {
                        'output': retrograde_res_update,
                        "fence_line": fence_line,
                    }

                    if retrograde_res is not None and len(retrograde_res) > 0:
                        print("retrograde res:", retrograde_res)

                    self.pipeline_res.update(retrograde_res_dict,
                                             'vehicle_retrograde')

                    if frame_id > self.warmup_frame:
                        self.pipe_timer.module_time['vehicle_retrograde'].end()

                    retrograde_traj_len = 0

Z
zhiboniu 已提交
1063
            self.collector.append(frame_id, self.pipeline_res)
1064 1065 1066 1067 1068 1069 1070

            if frame_id > self.warmup_frame:
                self.pipe_timer.img_num += 1
                self.pipe_timer.total_time.end()
            frame_id += 1

            if self.cfg['visual']:
1071
                _, _, fps = self.pipe_timer.get_total_time()
1072

1073
                im = self.visualize_video(frame_rgb, self.pipeline_res,
1074 1075 1076 1077
                                          self.collector, frame_id, fps,
                                          entrance, records, center_traj,
                                          self.illegal_parking_time != -1,
                                          illegal_parking_dict)  # visualize
L
LokeZhou 已提交
1078
                if len(self.pushurl) > 0:
Z
zhiboniu 已提交
1079 1080 1081 1082 1083 1084 1085
                    pushstream.pipe.stdin.write(im.tobytes())
                else:
                    writer.write(im)
                    if self.file_name is None:  # use camera_id
                        cv2.imshow('Paddle-Pipeline', im)
                        if cv2.waitKey(1) & 0xFF == ord('q'):
                            break
L
LokeZhou 已提交
1086 1087

        if self.cfg['visual'] and len(self.pushurl) == 0:
Z
zhiboniu 已提交
1088 1089
            writer.release()
            print('save result to {}'.format(out_path))
1090

1091
    def visualize_video(self,
1092
                        image_rgb,
1093
                        result,
1094
                        collector,
1095 1096 1097 1098
                        frame_id,
                        fps,
                        entrance=None,
                        records=None,
1099 1100 1101
                        center_traj=None,
                        do_illegal_parking_recognition=False,
                        illegal_parking_dict=None):
1102
        image = cv2.cvtColor(image_rgb, cv2.COLOR_RGB2BGR)
Z
zhiboniu 已提交
1103
        mot_res = copy.deepcopy(result.get('mot'))
L
LokeZhou 已提交
1104

1105 1106
        if mot_res is not None:
            ids = mot_res['boxes'][:, 0]
W
wangguanzhong 已提交
1107
            scores = mot_res['boxes'][:, 2]
1108 1109 1110 1111 1112 1113
            boxes = mot_res['boxes'][:, 3:]
            boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
            boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
        else:
            boxes = np.zeros([0, 4])
            ids = np.zeros([0])
W
wangguanzhong 已提交
1114
            scores = np.zeros([0])
1115 1116 1117 1118 1119 1120 1121 1122 1123 1124

        # single class, still need to be defaultdict type for ploting
        num_classes = 1
        online_tlwhs = defaultdict(list)
        online_scores = defaultdict(list)
        online_ids = defaultdict(list)
        online_tlwhs[0] = boxes
        online_scores[0] = scores
        online_ids[0] = ids

F
Feng Ni 已提交
1125 1126 1127 1128 1129 1130 1131 1132 1133
        if mot_res is not None:
            image = plot_tracking_dict(
                image,
                num_classes,
                online_tlwhs,
                online_ids,
                online_scores,
                frame_id=frame_id,
                fps=fps,
1134
                ids2names=self.mot_predictor.pred_config.labels,
F
Feng Ni 已提交
1135
                do_entrance_counting=self.do_entrance_counting,
1136
                do_break_in_counting=self.do_break_in_counting,
1137 1138
                do_illegal_parking_recognition=do_illegal_parking_recognition,
                illegal_parking_dict=illegal_parking_dict,
F
Feng Ni 已提交
1139 1140 1141
                entrance=entrance,
                records=records,
                center_traj=center_traj)
1142

1143 1144 1145 1146 1147 1148 1149 1150 1151
        human_attr_res = result.get('attr')
        if human_attr_res is not None:
            boxes = mot_res['boxes'][:, 1:]
            human_attr_res = human_attr_res['output']
            image = visualize_attr(image, human_attr_res, boxes)
            image = np.array(image)

        vehicle_attr_res = result.get('vehicle_attr')
        if vehicle_attr_res is not None:
1152
            boxes = mot_res['boxes'][:, 1:]
1153 1154
            vehicle_attr_res = vehicle_attr_res['output']
            image = visualize_attr(image, vehicle_attr_res, boxes)
1155 1156
            image = np.array(image)

L
LokeZhou 已提交
1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170
        lanes_res = result.get('lanes')
        if lanes_res is not None:
            lanes = lanes_res['output'][0]
            image = visualize_lane(image, lanes)
            image = np.array(image)

        vehiclepress_res = result.get('vehicle_press')
        if vehiclepress_res is not None:
            press_vehicle = vehiclepress_res['output']
            if len(press_vehicle) > 0:
                image = visualize_vehiclepress(
                    image, press_vehicle, threshold=self.cfg['crop_thresh'])
                image = np.array(image)

1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184
        if mot_res is not None:
            vehicleplate = False
            plates = []
            for trackid in mot_res['boxes'][:, 0]:
                plate = collector.get_carlp(trackid)
                if plate != None:
                    vehicleplate = True
                    plates.append(plate)
                else:
                    plates.append("")
            if vehicleplate:
                boxes = mot_res['boxes'][:, 1:]
                image = visualize_vehicleplate(image, plates, boxes)
                image = np.array(image)
Z
zhiboniu 已提交
1185

J
JYChen 已提交
1186 1187 1188 1189 1190 1191 1192 1193
        kpt_res = result.get('kpt')
        if kpt_res is not None:
            image = visualize_pose(
                image,
                kpt_res,
                visual_thresh=self.cfg['kpt_thresh'],
                returnimg=True)

1194
        video_action_res = result.get('video_action')
J
JYChen 已提交
1195
        if video_action_res is not None:
1196 1197 1198
            video_action_score = None
            if video_action_res and video_action_res["class"] == 1:
                video_action_score = video_action_res["score"]
1199 1200 1201
            mot_boxes = None
            if mot_res:
                mot_boxes = mot_res['boxes']
1202 1203
            image = visualize_action(
                image,
1204
                mot_boxes,
J
JYChen 已提交
1205
                action_visual_collector=None,
1206 1207 1208
                action_text="SkeletonAction",
                video_action_score=video_action_score,
                video_action_text="Fight")
J
JYChen 已提交
1209

L
LokeZhou 已提交
1210 1211 1212 1213 1214 1215 1216
        vehicle_retrograde_res = result.get('vehicle_retrograde')
        if vehicle_retrograde_res is not None:
            mot_retrograde_res = copy.deepcopy(result.get('mot'))
            image = visualize_vehicle_retrograde(image, mot_retrograde_res,
                                                 vehicle_retrograde_res)
            image = np.array(image)

J
JYChen 已提交
1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239
        visual_helper_for_display = []
        action_to_display = []

        skeleton_action_res = result.get('skeleton_action')
        if skeleton_action_res is not None:
            visual_helper_for_display.append(self.skeleton_action_visual_helper)
            action_to_display.append("Falling")

        det_action_res = result.get('det_action')
        if det_action_res is not None:
            visual_helper_for_display.append(self.det_action_visual_helper)
            action_to_display.append("Smoking")

        cls_action_res = result.get('cls_action')
        if cls_action_res is not None:
            visual_helper_for_display.append(self.cls_action_visual_helper)
            action_to_display.append("Calling")

        if len(visual_helper_for_display) > 0:
            image = visualize_action(image, mot_res['boxes'],
                                     visual_helper_for_display,
                                     action_to_display)

1240 1241 1242 1243 1244
        return image

    def visualize_image(self, im_files, images, result):
        start_idx, boxes_num_i = 0, 0
        det_res = result.get('det')
1245 1246
        human_attr_res = result.get('attr')
        vehicle_attr_res = result.get('vehicle_attr')
Z
zhiboniu 已提交
1247
        vehicleplate_res = result.get('vehicleplate')
L
LokeZhou 已提交
1248 1249
        lanes_res = result.get('lanes')
        vehiclepress_res = result.get('vehicle_press')
1250

1251 1252 1253 1254 1255 1256 1257 1258 1259
        for i, (im_file, im) in enumerate(zip(im_files, images)):
            if det_res is not None:
                det_res_i = {}
                boxes_num_i = det_res['boxes_num'][i]
                det_res_i['boxes'] = det_res['boxes'][start_idx:start_idx +
                                                      boxes_num_i, :]
                im = visualize_box_mask(
                    im,
                    det_res_i,
Z
zhiboniu 已提交
1260
                    labels=['target'],
1261
                    threshold=self.cfg['crop_thresh'])
1262 1263
                im = np.ascontiguousarray(np.copy(im))
                im = cv2.cvtColor(im, cv2.COLOR_RGB2BGR)
1264 1265 1266 1267 1268 1269 1270 1271
            if human_attr_res is not None:
                human_attr_res_i = human_attr_res['output'][start_idx:start_idx
                                                            + boxes_num_i]
                im = visualize_attr(im, human_attr_res_i, det_res_i['boxes'])
            if vehicle_attr_res is not None:
                vehicle_attr_res_i = vehicle_attr_res['output'][
                    start_idx:start_idx + boxes_num_i]
                im = visualize_attr(im, vehicle_attr_res_i, det_res_i['boxes'])
Z
zhiboniu 已提交
1272 1273 1274 1275 1276
            if vehicleplate_res is not None:
                plates = vehicleplate_res['vehicleplate']
                det_res_i['boxes'][:, 4:6] = det_res_i[
                    'boxes'][:, 4:6] - det_res_i['boxes'][:, 2:4]
                im = visualize_vehicleplate(im, plates, det_res_i['boxes'])
L
LokeZhou 已提交
1277 1278 1279 1280 1281 1282 1283 1284 1285 1286
            if vehiclepress_res is not None:
                press_vehicle = vehiclepress_res['output'][i]
                if len(press_vehicle) > 0:
                    im = visualize_vehiclepress(
                        im, press_vehicle, threshold=self.cfg['crop_thresh'])
                    im = np.ascontiguousarray(np.copy(im))
            if lanes_res is not None:
                lanes = lanes_res['output'][i]
                im = visualize_lane(im, lanes)
                im = np.ascontiguousarray(np.copy(im))
1287

1288 1289 1290 1291
            img_name = os.path.split(im_file)[-1]
            if not os.path.exists(self.output_dir):
                os.makedirs(self.output_dir)
            out_path = os.path.join(self.output_dir, img_name)
1292
            cv2.imwrite(out_path, im)
1293 1294 1295 1296 1297
            print("save result to: " + out_path)
            start_idx += boxes_num_i


def main():
1298
    cfg = merge_cfg(FLAGS)  # use command params to update config
1299
    print_arguments(cfg)
1300

Z
zhiboniu 已提交
1301
    pipeline = Pipeline(FLAGS, cfg)
1302 1303
    # pipeline.run()
    pipeline.run_multithreads()
1304 1305 1306 1307


if __name__ == '__main__':
    paddle.enable_static()
1308 1309

    # parse params from command
1310 1311 1312 1313 1314 1315 1316
    parser = argsparser()
    FLAGS = parser.parse_args()
    FLAGS.device = FLAGS.device.upper()
    assert FLAGS.device in ['CPU', 'GPU', 'XPU'
                            ], "device should be CPU, GPU or XPU"

    main()