pipeline.py 31.6 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import yaml
import glob
18
from collections import defaultdict
19 20 21 22 23 24

import cv2
import numpy as np
import math
import paddle
import sys
Z
zhiboniu 已提交
25
import copy
26
from collections import Sequence
Z
zhiboniu 已提交
27 28 29
from reid import ReID
from datacollector import DataCollector, Result
from mtmct import mtmct_process
30 31 32 33 34 35 36

# add deploy path of PadleDetection to sys.path
parent_path = os.path.abspath(os.path.join(__file__, *(['..'] * 2)))
sys.path.insert(0, parent_path)

from python.infer import Detector, DetectorPicoDet
from python.attr_infer import AttrDetector
J
JYChen 已提交
37 38
from python.keypoint_infer import KeyPointDetector
from python.keypoint_postprocess import translate_to_ori_images
39 40
from python.action_infer import FallingRecognizer
from python.action_utils import KeyPointBuff, FallingVisualHelper
J
JYChen 已提交
41

42
from pipe_utils import argsparser, print_arguments, merge_cfg, PipeTimer
J
JYChen 已提交
43
from pipe_utils import get_test_images, crop_image_with_det, crop_image_with_mot, parse_mot_res, parse_mot_keypoint
44
from python.preprocess import decode_image
J
JYChen 已提交
45
from python.visualize import visualize_box_mask, visualize_attr, visualize_pose, visualize_action
46 47

from pptracking.python.mot_sde_infer import SDE_Detector
48 49
from pptracking.python.mot.visualize import plot_tracking_dict
from pptracking.python.mot.utils import flow_statistic
50 51 52 53 54 55 56 57 58 59 60 61 62


class Pipeline(object):
    """
    Pipeline

    Args:
        cfg (dict): config of models in pipeline
        image_file (string|None): the path of image file, default as None
        image_dir (string|None): the path of image directory, if not None, 
            then all the images in directory will be predicted, default as None
        video_file (string|None): the path of video file, default as None
        camera_id (int): the device id of camera to predict, default as -1
W
wangguanzhong 已提交
63
        enable_attr (bool): whether use attribute recognition, default as false
64
        enable_falling (bool): whether use action recognition, default as false
65 66 67 68 69 70 71 72 73 74 75 76
        device (string): the device to predict, options are: CPU/GPU/XPU, 
            default as CPU
        run_mode (string): the mode of prediction, options are: 
            paddle/trt_fp32/trt_fp16, default as paddle
        trt_min_shape (int): min shape for dynamic shape in trt, default as 1
        trt_max_shape (int): max shape for dynamic shape in trt, default as 1280
        trt_opt_shape (int): opt shape for dynamic shape in trt, default as 640
        trt_calib_mode (bool): If the model is produced by TRT offline quantitative
            calibration, trt_calib_mode need to set True. default as False
        cpu_threads (int): cpu threads, default as 1
        enable_mkldnn (bool): whether to open MKLDNN, default as False
        output_dir (string): The path of output, default as 'output'
77 78 79 80 81
        draw_center_traj (bool): Whether drawing the trajectory of center, default as False
        secs_interval (int): The seconds interval to count after tracking, default as 10
        do_entrance_counting(bool): Whether counting the numbers of identifiers entering 
            or getting out from the entrance, default as False,only support single class
            counting in MOT.
82 83 84 85 86 87 88
    """

    def __init__(self,
                 cfg,
                 image_file=None,
                 image_dir=None,
                 video_file=None,
Z
zhiboniu 已提交
89
                 video_dir=None,
90
                 camera_id=-1,
W
wangguanzhong 已提交
91
                 enable_attr=False,
92 93
                 enable_falling=False,
                 enable_mtmct=False,
94 95 96 97 98 99 100 101
                 device='CPU',
                 run_mode='paddle',
                 trt_min_shape=1,
                 trt_max_shape=1280,
                 trt_opt_shape=640,
                 trt_calib_mode=False,
                 cpu_threads=1,
                 enable_mkldnn=False,
102 103 104 105
                 output_dir='output',
                 draw_center_traj=False,
                 secs_interval=10,
                 do_entrance_counting=False):
106
        self.multi_camera = False
107
        self.enable_mtmct = enable_mtmct
108
        self.is_video = False
Z
zhiboniu 已提交
109 110
        self.output_dir = output_dir
        self.vis_result = cfg['visual']
111
        self.input = self._parse_input(image_file, image_dir, video_file,
Z
zhiboniu 已提交
112
                                       video_dir, camera_id)
113
        if self.multi_camera:
114 115 116
            self.predictor = []
            for name in self.input:
                predictor_item = PipePredictor(
117 118 119
                    cfg,
                    is_video=True,
                    multi_camera=True,
W
wangguanzhong 已提交
120
                    enable_attr=enable_attr,
121 122
                    enable_falling=enable_falling,
                    enable_mtmct=enable_mtmct,
123 124 125 126 127 128 129
                    device=device,
                    run_mode=run_mode,
                    trt_min_shape=trt_min_shape,
                    trt_max_shape=trt_max_shape,
                    trt_opt_shape=trt_opt_shape,
                    cpu_threads=cpu_threads,
                    enable_mkldnn=enable_mkldnn,
130 131 132 133
                    output_dir=output_dir)
                predictor_item.set_file_name(name)
                self.predictor.append(predictor_item)

134 135 136 137
        else:
            self.predictor = PipePredictor(
                cfg,
                self.is_video,
W
wangguanzhong 已提交
138
                enable_attr=enable_attr,
139 140
                enable_falling=enable_falling,
                enable_mtmct=enable_mtmct,
141 142 143 144 145 146 147 148
                device=device,
                run_mode=run_mode,
                trt_min_shape=trt_min_shape,
                trt_max_shape=trt_max_shape,
                trt_opt_shape=trt_opt_shape,
                trt_calib_mode=trt_calib_mode,
                cpu_threads=cpu_threads,
                enable_mkldnn=enable_mkldnn,
149 150 151 152
                output_dir=output_dir,
                draw_center_traj=draw_center_traj,
                secs_interval=secs_interval,
                do_entrance_counting=do_entrance_counting)
153 154
            if self.is_video:
                self.predictor.set_file_name(video_file)
155

156 157 158 159 160
        self.output_dir = output_dir
        self.draw_center_traj = draw_center_traj
        self.secs_interval = secs_interval
        self.do_entrance_counting = do_entrance_counting

Z
zhiboniu 已提交
161 162
    def _parse_input(self, image_file, image_dir, video_file, video_dir,
                     camera_id):
163 164 165 166 167 168 169 170 171

        # parse input as is_video and multi_camera

        if image_file is not None or image_dir is not None:
            input = get_test_images(image_dir, image_file)
            self.is_video = False
            self.multi_camera = False

        elif video_file is not None:
172
            assert os.path.exists(video_file), "video_file not exists."
Z
zhiboniu 已提交
173 174 175 176 177 178 179
            self.multi_camera = False
            input = video_file
            self.is_video = True

        elif video_dir is not None:
            videof = [os.path.join(video_dir, x) for x in os.listdir(video_dir)]
            if len(videof) > 1:
180
                self.multi_camera = True
Z
zhiboniu 已提交
181 182
                videof.sort()
                input = videof
183
            else:
Z
zhiboniu 已提交
184
                input = videof[0]
185 186 187
            self.is_video = True

        elif camera_id != -1:
Z
zhiboniu 已提交
188 189
            self.multi_camera = False
            input = camera_id
190 191 192 193 194 195 196 197 198 199 200 201 202 203
            self.is_video = True

        else:
            raise ValueError(
                "Illegal Input, please set one of ['video_file','camera_id','image_file', 'image_dir']"
            )

        return input

    def run(self):
        if self.multi_camera:
            multi_res = []
            for predictor, input in zip(self.predictor, self.input):
                predictor.run(input)
Z
zhiboniu 已提交
204 205
                collector_data = predictor.get_result()
                multi_res.append(collector_data)
206 207 208 209 210 211
            if self.enable_mtmct:
                mtmct_process(
                    multi_res,
                    self.input,
                    mtmct_vis=self.vis_result,
                    output_dir=self.output_dir)
212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229

        else:
            self.predictor.run(self.input)


class PipePredictor(object):
    """
    Predictor in single camera
    
    The pipeline for image input: 

        1. Detection
        2. Detection -> Attribute

    The pipeline for video input: 

        1. Tracking
        2. Tracking -> Attribute
230
        3. Tracking -> KeyPoint -> Falling Recognition
231 232 233 234 235 236 237

    Args:
        cfg (dict): config of models in pipeline
        is_video (bool): whether the input is video, default as False
        multi_camera (bool): whether to use multi camera in pipeline, 
            default as False
        camera_id (int): the device id of camera to predict, default as -1
W
wangguanzhong 已提交
238
        enable_attr (bool): whether use attribute recognition, default as false
239
        enable_falling (bool): whether use action recognition, default as false
240 241 242 243 244 245 246 247 248 249 250 251
        device (string): the device to predict, options are: CPU/GPU/XPU, 
            default as CPU
        run_mode (string): the mode of prediction, options are: 
            paddle/trt_fp32/trt_fp16, default as paddle
        trt_min_shape (int): min shape for dynamic shape in trt, default as 1
        trt_max_shape (int): max shape for dynamic shape in trt, default as 1280
        trt_opt_shape (int): opt shape for dynamic shape in trt, default as 640
        trt_calib_mode (bool): If the model is produced by TRT offline quantitative
            calibration, trt_calib_mode need to set True. default as False
        cpu_threads (int): cpu threads, default as 1
        enable_mkldnn (bool): whether to open MKLDNN, default as False
        output_dir (string): The path of output, default as 'output'
252 253 254 255 256
        draw_center_traj (bool): Whether drawing the trajectory of center, default as False
        secs_interval (int): The seconds interval to count after tracking, default as 10
        do_entrance_counting(bool): Whether counting the numbers of identifiers entering 
            or getting out from the entrance, default as False,only support single class
            counting in MOT.
257 258 259 260 261 262
    """

    def __init__(self,
                 cfg,
                 is_video=True,
                 multi_camera=False,
W
wangguanzhong 已提交
263
                 enable_attr=False,
264 265
                 enable_falling=False,
                 enable_mtmct=False,
266 267 268 269 270 271 272 273
                 device='CPU',
                 run_mode='paddle',
                 trt_min_shape=1,
                 trt_max_shape=1280,
                 trt_opt_shape=640,
                 trt_calib_mode=False,
                 cpu_threads=1,
                 enable_mkldnn=False,
274 275 276 277
                 output_dir='output',
                 draw_center_traj=False,
                 secs_interval=10,
                 do_entrance_counting=False):
278

W
wangguanzhong 已提交
279 280 281
        if enable_attr and not cfg.get('ATTR', False):
            ValueError(
                'enable_attr is set to True, please set ATTR in config file')
282 283
        if enable_falling and (not cfg.get('FALLING', False) or
                               not cfg.get('KPT', False)):
W
wangguanzhong 已提交
284
            ValueError(
285
                'enable_falling is set to True, please set KPT and FALLING in config file'
W
wangguanzhong 已提交
286 287 288
            )

        self.with_attr = cfg.get('ATTR', False) and enable_attr
289 290
        self.with_falling = cfg.get('FALLING', False) and enable_falling
        self.with_mtmct = cfg.get('REID', False) and enable_mtmct
W
wangguanzhong 已提交
291 292
        if self.with_attr:
            print('Attribute Recognition enabled')
293 294 295
        if self.with_falling:
            print('Falling Recognition enabled')
        if enable_mtmct:
Z
zhiboniu 已提交
296 297 298 299 300 301
            if not self.with_mtmct:
                print(
                    'Warning!!! MTMCT enabled, but cannot find REID config in [infer_cfg.yml], please check!'
                )
            else:
                print("MTMCT enabled")
W
wangguanzhong 已提交
302

303 304 305 306 307 308
        self.modebase = {
            "framebased": False,
            "videobased": False,
            "idbased": False,
            "skeletonbased": False
        }
309 310 311 312
        self.is_video = is_video
        self.multi_camera = multi_camera
        self.cfg = cfg
        self.output_dir = output_dir
313 314 315
        self.draw_center_traj = draw_center_traj
        self.secs_interval = secs_interval
        self.do_entrance_counting = do_entrance_counting
316

J
JYChen 已提交
317
        self.warmup_frame = self.cfg['warmup_frame']
318 319
        self.pipeline_res = Result()
        self.pipe_timer = PipeTimer()
320
        self.file_name = None
Z
zhiboniu 已提交
321
        self.collector = DataCollector()
322 323 324 325 326 327 328 329 330 331 332 333 334

        if not is_video:
            det_cfg = self.cfg['DET']
            model_dir = det_cfg['model_dir']
            batch_size = det_cfg['batch_size']
            self.det_predictor = Detector(
                model_dir, device, run_mode, batch_size, trt_min_shape,
                trt_max_shape, trt_opt_shape, trt_calib_mode, cpu_threads,
                enable_mkldnn)
            if self.with_attr:
                attr_cfg = self.cfg['ATTR']
                model_dir = attr_cfg['model_dir']
                batch_size = attr_cfg['batch_size']
335 336
                basemode = attr_cfg['basemode']
                self.modebase[basemode] = True
337 338 339 340 341 342 343 344 345 346
                self.attr_predictor = AttrDetector(
                    model_dir, device, run_mode, batch_size, trt_min_shape,
                    trt_max_shape, trt_opt_shape, trt_calib_mode, cpu_threads,
                    enable_mkldnn)

        else:
            mot_cfg = self.cfg['MOT']
            model_dir = mot_cfg['model_dir']
            tracker_config = mot_cfg['tracker_config']
            batch_size = mot_cfg['batch_size']
347 348
            basemode = mot_cfg['basemode']
            self.modebase[basemode] = True
349
            self.mot_predictor = SDE_Detector(
350 351 352 353 354 355 356 357 358 359 360 361 362 363
                model_dir,
                tracker_config,
                device,
                run_mode,
                batch_size,
                trt_min_shape,
                trt_max_shape,
                trt_opt_shape,
                trt_calib_mode,
                cpu_threads,
                enable_mkldnn,
                draw_center_traj=draw_center_traj,
                secs_interval=secs_interval,
                do_entrance_counting=do_entrance_counting)
364 365 366 367
            if self.with_attr:
                attr_cfg = self.cfg['ATTR']
                model_dir = attr_cfg['model_dir']
                batch_size = attr_cfg['batch_size']
368 369
                basemode = attr_cfg['basemode']
                self.modebase[basemode] = True
370 371 372 373
                self.attr_predictor = AttrDetector(
                    model_dir, device, run_mode, batch_size, trt_min_shape,
                    trt_max_shape, trt_opt_shape, trt_calib_mode, cpu_threads,
                    enable_mkldnn)
374 375 376 377 378 379 380 381 382 383 384 385
            if self.with_falling:
                falling_cfg = self.cfg['FALLING']
                falling_model_dir = falling_cfg['model_dir']
                falling_batch_size = falling_cfg['batch_size']
                falling_frames = falling_cfg['max_frames']
                display_frames = falling_cfg['display_frames']
                self.coord_size = falling_cfg['coord_size']
                basemode = falling_cfg['basemode']
                self.modebase[basemode] = True

                self.falling_predictor = FallingRecognizer(
                    falling_model_dir,
J
JYChen 已提交
386 387
                    device,
                    run_mode,
388
                    falling_batch_size,
J
JYChen 已提交
389 390 391 392 393 394
                    trt_min_shape,
                    trt_max_shape,
                    trt_opt_shape,
                    trt_calib_mode,
                    cpu_threads,
                    enable_mkldnn,
395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414
                    window_size=falling_frames)
                self.falling_visual_helper = FallingVisualHelper(display_frames)

                if self.modebase["skeletonbased"]:
                    kpt_cfg = self.cfg['KPT']
                    kpt_model_dir = kpt_cfg['model_dir']
                    kpt_batch_size = kpt_cfg['batch_size']
                    self.kpt_predictor = KeyPointDetector(
                        kpt_model_dir,
                        device,
                        run_mode,
                        kpt_batch_size,
                        trt_min_shape,
                        trt_max_shape,
                        trt_opt_shape,
                        trt_calib_mode,
                        cpu_threads,
                        enable_mkldnn,
                        use_dark=False)
                    self.kpt_buff = KeyPointBuff(falling_frames)
Z
zhiboniu 已提交
415 416 417 418 419 420 421 422 423

        if self.with_mtmct:
            reid_cfg = self.cfg['REID']
            model_dir = reid_cfg['model_dir']
            batch_size = reid_cfg['batch_size']
            self.reid_predictor = ReID(model_dir, device, run_mode, batch_size,
                                       trt_min_shape, trt_max_shape,
                                       trt_opt_shape, trt_calib_mode,
                                       cpu_threads, enable_mkldnn)
424

425
    def set_file_name(self, path):
W
wangguanzhong 已提交
426 427 428 429 430
        if path is not None:
            self.file_name = os.path.split(path)[-1]
        else:
            # use camera id
            self.file_name = None
431

432
    def get_result(self):
Z
zhiboniu 已提交
433
        return self.collector.get_res()
434 435 436 437 438 439

    def run(self, input):
        if self.is_video:
            self.predict_video(input)
        else:
            self.predict_image(input)
440
        self.pipe_timer.info()
441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458

    def predict_image(self, input):
        # det
        # det -> attr
        batch_loop_cnt = math.ceil(
            float(len(input)) / self.det_predictor.batch_size)
        for i in range(batch_loop_cnt):
            start_index = i * self.det_predictor.batch_size
            end_index = min((i + 1) * self.det_predictor.batch_size, len(input))
            batch_file = input[start_index:end_index]
            batch_input = [decode_image(f, {})[0] for f in batch_file]

            if i > self.warmup_frame:
                self.pipe_timer.total_time.start()
                self.pipe_timer.module_time['det'].start()
            # det output format: class, score, xmin, ymin, xmax, ymax
            det_res = self.det_predictor.predict_image(
                batch_input, visual=False)
459 460
            det_res = self.det_predictor.filter_box(det_res,
                                                    self.cfg['crop_thresh'])
461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489
            if i > self.warmup_frame:
                self.pipe_timer.module_time['det'].end()
            self.pipeline_res.update(det_res, 'det')

            if self.with_attr:
                crop_inputs = crop_image_with_det(batch_input, det_res)
                attr_res_list = []

                if i > self.warmup_frame:
                    self.pipe_timer.module_time['attr'].start()

                for crop_input in crop_inputs:
                    attr_res = self.attr_predictor.predict_image(
                        crop_input, visual=False)
                    attr_res_list.extend(attr_res['output'])

                if i > self.warmup_frame:
                    self.pipe_timer.module_time['attr'].end()

                attr_res = {'output': attr_res_list}
                self.pipeline_res.update(attr_res, 'attr')

            self.pipe_timer.img_num += len(batch_input)
            if i > self.warmup_frame:
                self.pipe_timer.total_time.end()

            if self.cfg['visual']:
                self.visualize_image(batch_file, batch_input, self.pipeline_res)

Z
zhiboniu 已提交
490
    def predict_video(self, video_file):
491 492 493
        # mot
        # mot -> attr
        # mot -> pose -> action
Z
zhiboniu 已提交
494
        capture = cv2.VideoCapture(video_file)
495
        video_out_name = 'output.mp4' if self.file_name is None else self.file_name
496 497 498 499 500 501

        # Get Video info : resolution, fps, frame count
        width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH))
        height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT))
        fps = int(capture.get(cv2.CAP_PROP_FPS))
        frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT))
502
        print("video fps: %d, frame_count: %d" % (fps, frame_count))
503 504 505 506 507 508 509

        if not os.path.exists(self.output_dir):
            os.makedirs(self.output_dir)
        out_path = os.path.join(self.output_dir, video_out_name)
        fourcc = cv2.VideoWriter_fourcc(* 'mp4v')
        writer = cv2.VideoWriter(out_path, fourcc, fps, (width, height))
        frame_id = 0
510 511 512 513 514 515 516 517 518 519 520 521 522

        entrance, records, center_traj = None, None, None
        if self.draw_center_traj:
            center_traj = [{}]
        id_set = set()
        interval_id_set = set()
        in_id_list = list()
        out_id_list = list()
        prev_center = dict()
        records = list()
        entrance = [0, height / 2., width, height / 2.]
        video_fps = fps

523 524 525 526 527 528 529
        while (1):
            if frame_id % 10 == 0:
                print('frame id: ', frame_id)
            ret, frame = capture.read()
            if not ret:
                break

530
            if self.modebase["idbased"] or self.modebase["skeletonbased"]:
531
                if frame_id > self.warmup_frame:
532 533 534 535
                    self.pipe_timer.total_time.start()
                    self.pipe_timer.module_time['mot'].start()
                res = self.mot_predictor.predict_image(
                    [copy.deepcopy(frame)], visual=False)
536

J
JYChen 已提交
537
                if frame_id > self.warmup_frame:
538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555
                    self.pipe_timer.module_time['mot'].end()

                # mot output format: id, class, score, xmin, ymin, xmax, ymax
                mot_res = parse_mot_res(res)

                # flow_statistic only support single class MOT
                boxes, scores, ids = res[0]  # batch size = 1 in MOT
                mot_result = (frame_id + 1, boxes[0], scores[0],
                              ids[0])  # single class
                statistic = flow_statistic(
                    mot_result, self.secs_interval, self.do_entrance_counting,
                    video_fps, entrance, id_set, interval_id_set, in_id_list,
                    out_id_list, prev_center, records)
                records = statistic['records']

                # nothing detected
                if len(mot_res['boxes']) == 0:
                    frame_id += 1
J
JYChen 已提交
556
                    if frame_id > self.warmup_frame:
557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577
                        self.pipe_timer.img_num += 1
                        self.pipe_timer.total_time.end()
                    if self.cfg['visual']:
                        _, _, fps = self.pipe_timer.get_total_time()
                        im = self.visualize_video(frame, mot_res, frame_id, fps,
                                                  entrance, records,
                                                  center_traj)  # visualize
                        writer.write(im)
                        if self.file_name is None:  # use camera_id
                            cv2.imshow('PPHuman', im)
                            if cv2.waitKey(1) & 0xFF == ord('q'):
                                break

                    continue

                self.pipeline_res.update(mot_res, 'mot')
                if self.with_attr or self.with_falling:
                    crop_input, new_bboxes, ori_bboxes = crop_image_with_mot(
                        frame, mot_res)

                if self.with_attr:
J
JYChen 已提交
578
                    if frame_id > self.warmup_frame:
579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634
                        self.pipe_timer.module_time['attr'].start()
                    attr_res = self.attr_predictor.predict_image(
                        crop_input, visual=False)
                    if frame_id > self.warmup_frame:
                        self.pipe_timer.module_time['attr'].end()
                    self.pipeline_res.update(attr_res, 'attr')

                if self.with_falling:
                    if self.modebase["skeletonbased"]:
                        if frame_id > self.warmup_frame:
                            self.pipe_timer.module_time['kpt'].start()
                        kpt_pred = self.kpt_predictor.predict_image(
                            crop_input, visual=False)
                        keypoint_vector, score_vector = translate_to_ori_images(
                            kpt_pred, np.array(new_bboxes))
                        kpt_res = {}
                        kpt_res['keypoint'] = [
                            keypoint_vector.tolist(), score_vector.tolist()
                        ] if len(keypoint_vector) > 0 else [[], []]
                        kpt_res['bbox'] = ori_bboxes
                        if frame_id > self.warmup_frame:
                            self.pipe_timer.module_time['kpt'].end()

                        self.pipeline_res.update(kpt_res, 'kpt')

                        self.kpt_buff.update(kpt_res,
                                             mot_res)  # collect kpt output
                    state = self.kpt_buff.get_state(
                    )  # whether frame num is enough or lost tracker

                    falling_res = {}
                    if state:
                        if frame_id > self.warmup_frame:
                            self.pipe_timer.module_time['falling'].start()
                        collected_keypoint = self.kpt_buff.get_collected_keypoint(
                        )  # reoragnize kpt output with ID
                        falling_input = parse_mot_keypoint(collected_keypoint,
                                                           self.coord_size)
                        falling_res = self.falling_predictor.predict_skeleton_with_mot(
                            falling_input)
                        if frame_id > self.warmup_frame:
                            self.pipe_timer.module_time['falling'].end()
                        self.pipeline_res.update(falling_res, 'falling')

                    if self.cfg['visual']:
                        self.falling_visual_helper.update(falling_res)

                if self.with_mtmct and frame_id % 10 == 0:
                    crop_input, img_qualities, rects = self.reid_predictor.crop_image_with_mot(
                        frame, mot_res)
                    if frame_id > self.warmup_frame:
                        self.pipe_timer.module_time['reid'].start()
                    reid_res = self.reid_predictor.predict_batch(crop_input)

                    if frame_id > self.warmup_frame:
                        self.pipe_timer.module_time['reid'].end()
J
JYChen 已提交
635

636 637 638 639 640 641 642 643
                    reid_res_dict = {
                        'features': reid_res,
                        "qualities": img_qualities,
                        "rects": rects
                    }
                    self.pipeline_res.update(reid_res_dict, 'reid')
                else:
                    self.pipeline_res.clear('reid')
Z
zhiboniu 已提交
644

645 646
            if self.modebase["videobased"]:
                pass
Z
zhiboniu 已提交
647

648 649
            if self.modebase["framebased"]:
                pass
Z
zhiboniu 已提交
650 651

            self.collector.append(frame_id, self.pipeline_res)
652 653 654 655 656 657 658

            if frame_id > self.warmup_frame:
                self.pipe_timer.img_num += 1
                self.pipe_timer.total_time.end()
            frame_id += 1

            if self.cfg['visual']:
659 660
                _, _, fps = self.pipe_timer.get_total_time()
                im = self.visualize_video(frame, self.pipeline_res, frame_id,
661 662
                                          fps, entrance, records,
                                          center_traj)  # visualize
663
                writer.write(im)
W
wangguanzhong 已提交
664 665 666 667
                if self.file_name is None:  # use camera_id
                    cv2.imshow('PPHuman', im)
                    if cv2.waitKey(1) & 0xFF == ord('q'):
                        break
668 669 670 671

        writer.release()
        print('save result to {}'.format(out_path))

672 673 674 675 676 677 678 679
    def visualize_video(self,
                        image,
                        result,
                        frame_id,
                        fps,
                        entrance=None,
                        records=None,
                        center_traj=None):
Z
zhiboniu 已提交
680
        mot_res = copy.deepcopy(result.get('mot'))
681 682
        if mot_res is not None:
            ids = mot_res['boxes'][:, 0]
W
wangguanzhong 已提交
683
            scores = mot_res['boxes'][:, 2]
684 685 686 687 688 689
            boxes = mot_res['boxes'][:, 3:]
            boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
            boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
        else:
            boxes = np.zeros([0, 4])
            ids = np.zeros([0])
W
wangguanzhong 已提交
690
            scores = np.zeros([0])
691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712

        # single class, still need to be defaultdict type for ploting
        num_classes = 1
        online_tlwhs = defaultdict(list)
        online_scores = defaultdict(list)
        online_ids = defaultdict(list)
        online_tlwhs[0] = boxes
        online_scores[0] = scores
        online_ids[0] = ids

        image = plot_tracking_dict(
            image,
            num_classes,
            online_tlwhs,
            online_ids,
            online_scores,
            frame_id=frame_id,
            fps=fps,
            do_entrance_counting=self.do_entrance_counting,
            entrance=entrance,
            records=records,
            center_traj=center_traj)
713 714 715 716 717 718 719 720

        attr_res = result.get('attr')
        if attr_res is not None:
            boxes = mot_res['boxes'][:, 1:]
            attr_res = attr_res['output']
            image = visualize_attr(image, attr_res, boxes)
            image = np.array(image)

J
JYChen 已提交
721 722 723 724 725 726 727 728
        kpt_res = result.get('kpt')
        if kpt_res is not None:
            image = visualize_pose(
                image,
                kpt_res,
                visual_thresh=self.cfg['kpt_thresh'],
                returnimg=True)

729 730
        falling_res = result.get('falling')
        if falling_res is not None:
J
JYChen 已提交
731
            image = visualize_action(image, mot_res['boxes'],
732
                                     self.falling_visual_helper, "Falling")
J
JYChen 已提交
733

734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750
        return image

    def visualize_image(self, im_files, images, result):
        start_idx, boxes_num_i = 0, 0
        det_res = result.get('det')
        attr_res = result.get('attr')
        for i, (im_file, im) in enumerate(zip(im_files, images)):
            if det_res is not None:
                det_res_i = {}
                boxes_num_i = det_res['boxes_num'][i]
                det_res_i['boxes'] = det_res['boxes'][start_idx:start_idx +
                                                      boxes_num_i, :]
                im = visualize_box_mask(
                    im,
                    det_res_i,
                    labels=['person'],
                    threshold=self.cfg['crop_thresh'])
751 752
                im = np.ascontiguousarray(np.copy(im))
                im = cv2.cvtColor(im, cv2.COLOR_RGB2BGR)
753 754 755 756 757 758 759 760
            if attr_res is not None:
                attr_res_i = attr_res['output'][start_idx:start_idx +
                                                boxes_num_i]
                im = visualize_attr(im, attr_res_i, det_res_i['boxes'])
            img_name = os.path.split(im_file)[-1]
            if not os.path.exists(self.output_dir):
                os.makedirs(self.output_dir)
            out_path = os.path.join(self.output_dir, img_name)
761
            cv2.imwrite(out_path, im)
762 763 764 765 766 767 768 769 770
            print("save result to: " + out_path)
            start_idx += boxes_num_i


def main():
    cfg = merge_cfg(FLAGS)
    print_arguments(cfg)
    pipeline = Pipeline(
        cfg, FLAGS.image_file, FLAGS.image_dir, FLAGS.video_file,
Z
zhiboniu 已提交
771
        FLAGS.video_dir, FLAGS.camera_id, FLAGS.enable_attr,
772 773 774 775 776
        FLAGS.enable_falling, FLAGS.enable_mtmct, FLAGS.device, FLAGS.run_mode,
        FLAGS.trt_min_shape, FLAGS.trt_max_shape, FLAGS.trt_opt_shape,
        FLAGS.trt_calib_mode, FLAGS.cpu_threads, FLAGS.enable_mkldnn,
        FLAGS.output_dir, FLAGS.draw_center_traj, FLAGS.secs_interval,
        FLAGS.do_entrance_counting)
777 778 779 780 781 782 783 784 785 786 787 788 789

    pipeline.run()


if __name__ == '__main__':
    paddle.enable_static()
    parser = argsparser()
    FLAGS = parser.parse_args()
    FLAGS.device = FLAGS.device.upper()
    assert FLAGS.device in ['CPU', 'GPU', 'XPU'
                            ], "device should be CPU, GPU or XPU"

    main()