From c8f5e3be80865c26f26b4b9f860c1e8d6d1bc77a Mon Sep 17 00:00:00 2001 From: zhiboniu <31800336+zhiboniu@users.noreply.github.com> Date: Tue, 27 Sep 2022 11:49:12 +0800 Subject: [PATCH] rtsp push stream in pipeline (#7000) * add push stream * update push stream docs * update class name --- deploy/pipeline/cfg_utils.py | 5 ++ .../docs/tutorials/PPHuman_QUICK_STARTED.md | 18 +++++- .../tutorials/PPHuman_QUICK_STARTED_en.md | 20 +++++- .../docs/tutorials/PPVehicle_QUICK_STARTED.md | 21 ++++++- .../tutorials/PPVehicle_QUICK_STARTED_en.md | 19 +++++- deploy/pipeline/pipe_utils.py | 21 +++++++ deploy/pipeline/pipeline.py | 63 ++++++++++++------- 7 files changed, 138 insertions(+), 29 deletions(-) diff --git a/deploy/pipeline/cfg_utils.py b/deploy/pipeline/cfg_utils.py index 37c963299..1ad6ed999 100644 --- a/deploy/pipeline/cfg_utils.py +++ b/deploy/pipeline/cfg_utils.py @@ -85,6 +85,11 @@ def argsparser(): type=str, default="output", help="Directory of output visualization files.") + parser.add_argument( + "--pushurl", + type=str, + default="", + help="url of output visualization stream.") parser.add_argument( "--run_mode", type=str, diff --git a/deploy/pipeline/docs/tutorials/PPHuman_QUICK_STARTED.md b/deploy/pipeline/docs/tutorials/PPHuman_QUICK_STARTED.md index 80e321764..ca8d6584e 100644 --- a/deploy/pipeline/docs/tutorials/PPHuman_QUICK_STARTED.md +++ b/deploy/pipeline/docs/tutorials/PPHuman_QUICK_STARTED.md @@ -125,7 +125,10 @@ python deploy/pipeline/pipeline.py --config deploy/pipeline/config/infer_cfg_pph python deploy/pipeline/pipeline.py --config deploy/pipeline/config/infer_cfg_pphuman.yml -o SKELETON_ACTION.enbale=True --video_file=test_video.mp4 --device=gpu ``` -3. 对rtsp流的支持，使用--rtsp RTSP [RTSP ...]参数指定一路或者多路rtsp视频流，如果是多路地址中间用空格隔开。(或者video_file后面的视频地址直接更换为rtsp流地址)，示例如下： +3. rtsp推拉流 +- rtsp拉流预测 + +对rtsp拉流的支持，使用--rtsp RTSP [RTSP ...]参数指定一路或者多路rtsp视频流，如果是多路地址中间用空格隔开。(或者video_file后面的视频地址直接更换为rtsp流地址)，示例如下： ``` # 例：行人属性识别，单路视频流 python deploy/pipeline/pipeline.py --config deploy/pipeline/config/examples/infer_cfg_human_attr.yml -o visual=False --rtsp rtsp://[YOUR_RTSP_SITE] --device=gpu @@ -134,6 +137,18 @@ python deploy/pipeline/pipeline.py --config deploy/pipeline/config/examples/infe python deploy/pipeline/pipeline.py --config deploy/pipeline/config/examples/infer_cfg_human_attr.yml -o visual=False --rtsp rtsp://[YOUR_RTSP_SITE1] rtsp://[YOUR_RTSP_SITE2] --device=gpu ``` +- 视频结果推流rtsp + +预测结果进行rtsp推流，使用--pushurl rtsp:[IP] 推流到IP地址端，PC端可以使用[VLC播放器](https://vlc.onl/)打开网络流进行播放，播放地址为 `rtsp:[IP]/videoname`。其中`videoname`是预测的视频文件名，如果视频来源是本地摄像头则`videoname`默认为`output`. +``` +# 例：行人属性识别，单路视频流，该示例播放地址为 rtsp://[YOUR_SERVER_IP]:8554/test_video +python deploy/pipeline/pipeline.py --config deploy/pipeline/config/examples/infer_cfg_human_attr.yml --video_file=test_video.mp4 --device=gpu --pushurl rtsp://[YOUR_SERVER_IP]:8554 +``` +注： +1. rtsp推流服务基于 [rtsp-simple-server](https://github.com/aler9/rtsp-simple-server), 如使用推流功能请先开启该服务. +2. rtsp推流如果模型处理速度跟不上会出现很明显的卡顿现象，建议跟踪模型使用ppyoloe_s版本，即修改配置中跟踪模型mot_ppyoloe_l_36e_pipeline.zip替换为mot_ppyoloe_s_36e_pipeline.zip。 + + ### Jetson部署说明 由于Jetson平台算力相比服务器有较大差距，有如下使用建议： @@ -158,6 +173,7 @@ python deploy/pipeline/pipeline.py --config deploy/pipeline/config/examples/infe | --rtsp | Option | rtsp视频流地址，支持一路或者多路同时输入 | | --camera_id | Option | 用来预测的摄像头ID，默认为-1(表示不使用摄像头预测，可设置为：0 - (摄像头数目-1) )，预测过程中在可视化界面按`q`退出输出预测结果到：output/output.mp4| | --device | Option | 运行时的设备，可选择`CPU/GPU/XPU`，默认为`CPU`| +| --pushurl | Option| 对预测结果视频进行推流的地址，以rtsp://开头，该选项优先级高于视频结果本地存储，打开时不再另外存储本地预测结果视频| | --output_dir | Option|可视化结果保存的根目录，默认为output/| | --run_mode | Option |使用GPU时，默认为paddle, 可选（paddle/trt_fp32/trt_fp16/trt_int8）| | --enable_mkldnn | Option | CPU预测中是否开启MKLDNN加速，默认为False | diff --git a/deploy/pipeline/docs/tutorials/PPHuman_QUICK_STARTED_en.md b/deploy/pipeline/docs/tutorials/PPHuman_QUICK_STARTED_en.md index 8e0ffa380..1a3d933bf 100644 --- a/deploy/pipeline/docs/tutorials/PPHuman_QUICK_STARTED_en.md +++ b/deploy/pipeline/docs/tutorials/PPHuman_QUICK_STARTED_en.md @@ -125,7 +125,10 @@ python deploy/pipeline/pipeline.py --config deploy/pipeline/config/infer_cfg_pph python deploy/pipeline/pipeline.py --config deploy/pipeline/config/infer_cfg_pphuman.yml -o SKELETON_ACTION.enbale=True --video_file=test_video.mp4 --device=gpu ``` -3. For rtsp stream, use --rtsp RTSP [RTSP ...] parameter to specify one or more rtsp streams. Separate the multiple addresses with a space, or replace the video address directly after the video_file with the rtsp stream address), examples as follows +3. rtsp push/pull stream +- rtsp pull stream + +For rtsp pull stream, use `--rtsp RTSP [RTSP ...]` parameter to specify one or more rtsp streams. Separate the multiple addresses with a space, or replace the video address directly after the video_file with the rtsp stream address), examples as follows ``` # Example: Single video stream for pedestrian attribute recognition @@ -134,6 +137,19 @@ python deploy/pipeline/pipeline.py --config deploy/pipeline/config/examples/infe python deploy/pipeline/pipeline.py --config deploy/pipeline/config/examples/infer_cfg_human_attr.yml -o visual=False --rtsp rtsp://[YOUR_RTSP_SITE1] rtsp://[YOUR_RTSP_SITE2] --device=gpu | ``` +- rtsp push stream + +For rtsp push stream, use `--pushurl rtsp:[IP]` parameter to push stream to a IP set, and you can visualize the output video by [VLC Player](https://vlc.onl/) with the `open network` funciton. the whole url path is `rtsp:[IP]/videoname`, the videoname here is the basename of the video file to infer, and the default of videoname is `output` when the video come from local camera and there is no video name. + +``` +# Example：Pedestrian attribute recognition，in this example the whole url path is: rtsp://[YOUR_SERVER_IP]:8554/test_video +python deploy/pipeline/pipeline.py --config deploy/pipeline/config/examples/infer_cfg_human_attr.yml --video_file=test_video.mp4 --device=gpu --pushurl rtsp://[YOUR_SERVER_IP]:8554 +``` +Note: +1. rtsp push stream is based on [rtsp-simple-server](https://github.com/aler9/rtsp-simple-server), please enable this serving first. +2. the output visualize will be frozen frequently if the model cost too much time, we suggest to use faster model like ppyoloe_s in tracking, this is simply replace mot_ppyoloe_l_36e_pipeline.zip with mot_ppyoloe_s_36e_pipeline.zip in model config yaml file. + + ### Jetson Deployment Due to the large gap in computing power of the Jetson platform compared to the server, we suggest: @@ -155,6 +171,8 @@ With this recommended configuration, it is possible to achieve higher speeds on | --rtsp | Option | rtsp video stream address, supports one or more simultaneous streams input | | --camera_id | Option | The camera ID for prediction, default is -1 ( for no camera prediction, can be set to 0 - (number of cameras - 1) ), press `q` in the visualization interface during the prediction process to output the prediction result to: output/output.mp4 | | --device | Option | Running device, options include `CPU/GPU/XPU`, and the default is `CPU`. | +| --pushurl | Option | push the output video to rtsp stream, normaly start with `rtsp://`; this has higher priority than local video save, while this is set, pipeline will not save local visualize video, the default is "", means this will not work now. + | | --output_dir | Option | The root directory for the visualization results, and the default is output/ | | --run_mode | Option | For GPU, the default is paddle, with (paddle/trt_fp32/trt_fp16/trt_int8) as optional | | --enable_mkldnn | Option | Whether to enable MKLDNN acceleration in CPU prediction, the default is False | diff --git a/deploy/pipeline/docs/tutorials/PPVehicle_QUICK_STARTED.md b/deploy/pipeline/docs/tutorials/PPVehicle_QUICK_STARTED.md index 325a4903f..bb34ed56c 100644 --- a/deploy/pipeline/docs/tutorials/PPVehicle_QUICK_STARTED.md +++ b/deploy/pipeline/docs/tutorials/PPVehicle_QUICK_STARTED.md @@ -131,15 +131,29 @@ python deploy/pipeline/pipeline.py --config deploy/pipeline/config/examples/infe ``` -3. 对rtsp流的支持，使用--rtsp RTSP [RTSP ...]参数指定一路或者多路rtsp视频流，如果是多路地址中间用空格隔开。(或者video_file后面的视频地址直接更换为rtsp流地址)，示例如下： +3. rtsp推拉流 +- rtsp拉流预测 + +对rtsp拉流的支持，使用--rtsp RTSP [RTSP ...]参数指定一路或者多路rtsp视频流，如果是多路地址中间用空格隔开。(或者video_file后面的视频地址直接更换为rtsp流地址)，示例如下： ``` -# 例：行人属性识别，单路视频流 +# 例：车辆属性识别，单路视频流 python deploy/pipeline/pipeline.py --config deploy/pipeline/config/examples/infer_cfg_vehicle_attr.yml -o visual=False --rtsp rtsp://[YOUR_RTSP_SITE] --device=gpu -# 例：行人属性识别，多路视频流 +# 例：车辆属性识别，多路视频流 python deploy/pipeline/pipeline.py --config deploy/pipeline/config/examples/infer_cfg_vehicle_attr.yml -o visual=False --rtsp rtsp://[YOUR_RTSP_SITE1] rtsp://[YOUR_RTSP_SITE2] --device=gpu ``` +- 视频结果推流rtsp + +预测结果进行rtsp推流，使用--pushurl rtsp:[IP] 推流到IP地址端，PC端可以使用[VLC播放器](https://vlc.onl/)打开网络流进行播放，播放地址为 `rtsp:[IP]/videoname`。其中`videoname`是预测的视频文件名，如果视频来源是本地摄像头则`videoname`默认为`output`. +``` +# 例：车辆属性识别，单路视频流，该示例播放地址为 rtsp://[YOUR_SERVER_IP]:8554/test_video +python deploy/pipeline/pipeline.py --config deploy/pipeline/config/examples/infer_cfg_vehicle_attr.yml -o visual=False --video_file=test_video.mp4 --device=gpu --pushurl rtsp://[YOUR_SERVER_IP]:8554 +``` +注： +1. rtsp推流服务基于 [rtsp-simple-server](https://github.com/aler9/rtsp-simple-server), 如使用推流功能请先开启该服务. +2. rtsp推流如果模型处理速度跟不上会出现很明显的卡顿现象，建议跟踪模型使用ppyoloe_s版本，即修改配置中跟踪模型mot_ppyoloe_l_36e_pipeline.zip替换为mot_ppyoloe_s_36e_pipeline.zip。 + ### Jetson部署说明 由于Jetson平台算力相比服务器有较大差距，有如下使用建议： @@ -164,6 +178,7 @@ python deploy/pipeline/pipeline.py --config deploy/pipeline/config/examples/infe | --rtsp | Option | rtsp视频流地址，支持一路或者多路同时输入 | | --camera_id | Option | 用来预测的摄像头ID，默认为-1(表示不使用摄像头预测，可设置为：0 - (摄像头数目-1) )，预测过程中在可视化界面按`q`退出输出预测结果到：output/output.mp4| | --device | Option | 运行时的设备，可选择`CPU/GPU/XPU`，默认为`CPU`| +| --pushurl | Option| 对预测结果视频进行推流的地址，以rtsp://开头，该选项优先级高于视频结果本地存储，打开时不再另外存储本地预测结果视频, 默认为空，表示没有开启| | --output_dir | Option|可视化结果保存的根目录，默认为output/| | --run_mode | Option |使用GPU时，默认为paddle, 可选（paddle/trt_fp32/trt_fp16/trt_int8）| | --enable_mkldnn | Option | CPU预测中是否开启MKLDNN加速，默认为False | diff --git a/deploy/pipeline/docs/tutorials/PPVehicle_QUICK_STARTED_en.md b/deploy/pipeline/docs/tutorials/PPVehicle_QUICK_STARTED_en.md index dc42b5056..be570cafe 100644 --- a/deploy/pipeline/docs/tutorials/PPVehicle_QUICK_STARTED_en.md +++ b/deploy/pipeline/docs/tutorials/PPVehicle_QUICK_STARTED_en.md @@ -129,7 +129,10 @@ python deploy/pipeline/pipeline.py --config deploy/pipeline/config/examples/infe --region_polygon 600 300 1300 300 1300 800 600 800 ``` -3. For rtsp stream, use --rtsp RTSP [RTSP ...] parameter to specify one or more rtsp streams. Separate the multiple addresses with a space, or replace the video address directly after the video_file with the rtsp stream address), examples as follows +3. rtsp push/pull stream +- rtsp pull stream + +For rtsp pull stream, use --rtsp RTSP [RTSP ...] parameter to specify one or more rtsp streams. Separate the multiple addresses with a space, or replace the video address directly after the video_file with the rtsp stream address), examples as follows ``` # Example: Single video stream for pedestrian attribute recognition @@ -138,6 +141,18 @@ python deploy/pipeline/pipeline.py --config deploy/pipeline/config/examples/infe python deploy/pipeline/pipeline.py --config deploy/pipeline/config/examples/infer_cfg_human_attr.yml -o visual=False --rtsp rtsp://[YOUR_RTSP_SITE1] rtsp://[YOUR_RTSP_SITE2] --device=gpu | ``` +- rtsp push stream + +For rtsp push stream, use --pushurl rtsp:[IP] parameter to push stream to a IP set, and you can visualize the output video by [VLC Player](https://vlc.onl/) with the `open network` funciton. the whole url path is `rtsp:[IP]/videoname`, the videoname here is the basename of the video file to infer, and the default of videoname is `output` when the video come from local camera and there is no video name. + +``` +# Example：license plate recognition，in this example the whole url path is: rtsp://[YOUR_SERVER_IP]:8554/test_video +python deploy/pipeline/pipeline.py --config deploy/pipeline/config/examples/infer_cfg_vehicle_plate.yml --video_file=test_video.mp4 --device=gpu --pushurl rtsp://[YOUR_SERVER_IP]:8554 +``` +Note: +1. rtsp push stream is based on [rtsp-simple-server](https://github.com/aler9/rtsp-simple-server), please enable this serving first. +2. the output visualize will be frozen frequently if the model cost too much time, we suggest to use faster model like ppyoloe_s in tracking, this is simply replace mot_ppyoloe_l_36e_pipeline.zip with mot_ppyoloe_s_36e_pipeline.zip in model config yaml file. + ### Jetson Deployment Due to the large gap in computing power of the Jetson platform compared to the server, we suggest: @@ -161,6 +176,8 @@ With this recommended configuration, it is possible to achieve higher speeds on | --rtsp | Option | rtsp video stream address, supports one or more simultaneous streams input | | --camera_id | Option | The camera ID for prediction, default is -1 ( for no camera prediction, can be set to 0 - (number of cameras - 1) ), press `q` in the visualization interface during the prediction process to output the prediction result to: output/output.mp4 | | --device | Option | Running device, options include `CPU/GPU/XPU`, and the default is `CPU`. | +| --pushurl | Option | push the output video to rtsp stream, normaly start with `rtsp://`; this has higher priority than local video save, while this is set, pipeline will not save local visualize video, the default is "", means this will not work now. + | | --output_dir | Option | The root directory for the visualization results, and the default is output/ | | --run_mode | Option | For GPU, the default is paddle, with (paddle/trt_fp32/trt_fp16/trt_int8) as optional | | --enable_mkldnn | Option | Whether to enable MKLDNN acceleration in CPU prediction, the default is False | diff --git a/deploy/pipeline/pipe_utils.py b/deploy/pipeline/pipe_utils.py index e17f8aeab..996a76a64 100644 --- a/deploy/pipeline/pipe_utils.py +++ b/deploy/pipeline/pipe_utils.py @@ -19,6 +19,7 @@ import glob import yaml import copy import numpy as np +import subprocess as sp from python.keypoint_preprocess import EvalAffine, TopDownEvalAffine, expand_crop @@ -122,6 +123,26 @@ class PipeTimer(Times): dic['img_num'] = self.img_num return dic +class PushStream(object): + def __init__(self, pushurl = "rtsp://127.0.0.1:8554/"): + self.command = "" + # 自行设置 + self.pushurl = pushurl + + def initcmd(self, fps, width, height): + self.command = ['ffmpeg', + '-y', + '-f', 'rawvideo', + '-vcodec','rawvideo', + '-pix_fmt', 'bgr24', + '-s', "{}x{}".format(width, height), + '-r', str(fps), + '-i', '-', + '-pix_fmt', 'yuv420p', + '-f', 'rtsp', + self.pushurl] + self.pipe = sp.Popen(self.command, stdin=sp.PIPE) + def get_test_images(infer_dir, infer_img): """ diff --git a/deploy/pipeline/pipeline.py b/deploy/pipeline/pipeline.py index 8865d5d75..b3e7892aa 100644 --- a/deploy/pipeline/pipeline.py +++ b/deploy/pipeline/pipeline.py @@ -31,6 +31,7 @@ sys.path.insert(0, parent_path) from cfg_utils import argsparser, print_arguments, merge_cfg from pipe_utils import PipeTimer from pipe_utils import get_test_images, crop_image_with_det, crop_image_with_mot, parse_mot_res, parse_mot_keypoint +from pipe_utils import PushStream from python.infer import Detector, DetectorPicoDet from python.keypoint_infer import KeyPointDetector @@ -340,6 +341,8 @@ class PipePredictor(object): self.file_name = None self.collector = DataCollector() + self.pushurl = args.pushurl + # auto download inference model get_model_dir(self.cfg) @@ -470,7 +473,7 @@ class PipePredictor(object): def set_file_name(self, path): if path is not None: - self.file_name = os.path.split(path)[-1] + self.file_name = os.path.split(path)[-1].split(".")[-2] else: # use camera id self.file_name = None @@ -571,10 +574,6 @@ class PipePredictor(object): # mot -> attr # mot -> pose -> action capture = cv2.VideoCapture(video_file) - video_out_name = 'output.mp4' if self.file_name is None else self.file_name - if "rtsp" in video_file: - video_out_name = video_out_name + "_t" + str(thread_idx).zfill( - 2) + "_rtsp.mp4" # Get Video info : resolution, fps, frame count width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH)) @@ -583,11 +582,23 @@ class PipePredictor(object): frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT)) print("video fps: %d, frame_count: %d" % (fps, frame_count)) - if not os.path.exists(self.output_dir): - os.makedirs(self.output_dir) - out_path = os.path.join(self.output_dir, video_out_name) - fourcc = cv2.VideoWriter_fourcc(* 'mp4v') - writer = cv2.VideoWriter(out_path, fourcc, fps, (width, height)) + if len(self.pushurl) > 0: + video_out_name = 'output' if self.file_name is None else self.file_name + pushurl = os.path.join(self.pushurl, video_out_name) + print("the result will push stream to url:{}".format(pushurl)) + pushstream = PushStream(pushurl) + pushstream.initcmd(fps, width, height) + elif self.cfg['visual']: + video_out_name = 'output' if self.file_name is None else self.file_name + if "rtsp" in video_file: + video_out_name = video_out_name + "_t" + str(thread_idx).zfill( + 2) + "_rtsp" + if not os.path.exists(self.output_dir): + os.makedirs(self.output_dir) + out_path = os.path.join(self.output_dir, video_out_name+".mp4") + fourcc = cv2.VideoWriter_fourcc(* 'mp4v') + writer = cv2.VideoWriter(out_path, fourcc, fps, (width, height)) + frame_id = 0 entrance, records, center_traj = None, None, None @@ -709,11 +720,14 @@ class PipePredictor(object): im = self.visualize_video(frame, mot_res, frame_id, fps, entrance, records, center_traj) # visualize - writer.write(im) - if self.file_name is None: # use camera_id - cv2.imshow('Paddle-Pipeline', im) - if cv2.waitKey(1) & 0xFF == ord('q'): - break + if len(self.pushurl)>0: + pushstream.pipe.stdin.write(im.tobytes()) + else: + writer.write(im) + if self.file_name is None: # use camera_id + cv2.imshow('Paddle-Pipeline', im) + if cv2.waitKey(1) & 0xFF == ord('q'): + break continue self.pipeline_res.update(mot_res, 'mot') @@ -882,14 +896,17 @@ class PipePredictor(object): entrance, records, center_traj, self.illegal_parking_time != -1, illegal_parking_dict) # visualize - writer.write(im) - if self.file_name is None: # use camera_id - cv2.imshow('Paddle-Pipeline', im) - if cv2.waitKey(1) & 0xFF == ord('q'): - break - - writer.release() - print('save result to {}'.format(out_path)) + if len(self.pushurl)>0: + pushstream.pipe.stdin.write(im.tobytes()) + else: + writer.write(im) + if self.file_name is None: # use camera_id + cv2.imshow('Paddle-Pipeline', im) + if cv2.waitKey(1) & 0xFF == ord('q'): + break + if self.cfg['visual'] and len(self.pushurl)==0: + writer.release() + print('save result to {}'.format(out_path)) def visualize_video(self, image, -- GitLab