From e301f5677b69f378d52ba74b3da01250810e7b98 Mon Sep 17 00:00:00 2001 From: jm12138 <2286040843@qq.com> Date: Thu, 29 Dec 2022 10:25:53 +0800 Subject: [PATCH] update humanseg_lite (#2181) --- .../humanseg_lite/README.md | 17 +- .../humanseg_lite/README_en.md | 24 +-- .../humanseg_lite/module.py | 151 +++++++++--------- .../humanseg_lite/optimal.py | 4 +- .../humanseg_lite/processor.py | 2 +- .../humanseg_lite/test.py | 98 ++++-------- 6 files changed, 136 insertions(+), 160 deletions(-) diff --git a/modules/image/semantic_segmentation/humanseg_lite/README.md b/modules/image/semantic_segmentation/humanseg_lite/README.md index 6f8cd16c..3bd4bc57 100644 --- a/modules/image/semantic_segmentation/humanseg_lite/README.md +++ b/modules/image/semantic_segmentation/humanseg_lite/README.md @@ -1,7 +1,7 @@ # humanseg_lite |模型名称|humanseg_lite| -| :--- | :---: | +| :--- | :---: | |类别|图像-图像分割| |网络|shufflenet| |数据集|百度自建数据集| @@ -13,7 +13,7 @@ ## 一、模型基本信息 - ### 应用效果展示 - + - 样例结果示例：

@@ -37,7 +37,7 @@ - ```shell $ hub install humanseg_lite ``` - + - 如您安装时遇到问题，可参考：[零基础windows安装](../../../../docs/docs_ch/get_start/windows_quickstart.md) | [零基础Linux安装](../../../../docs/docs_ch/get_start/linux_quickstart.md) | [零基础MacOS安装](../../../../docs/docs_ch/get_start/mac_quickstart.md) @@ -226,6 +226,9 @@ cv2.imwrite("segment_human_lite.png", rgba) ``` +- ### Gradio APP 支持 + + 从 PaddleHub 2.3.1 开始支持使用链接 http://127.0.0.1:8866/gradio/humanseg_lite 在浏览器中访问 humanseg_lite 的 Gradio APP。 ## 五、更新历史 @@ -234,7 +237,7 @@ 初始发布 * 1.1.0 - + 新增视频人像分割接口 新增视频流人像分割接口 @@ -247,6 +250,10 @@ 移除 Fluid API +* 1.3.0 + + 添加 Gradio APP 支持 + ```shell - $ hub install humanseg_lite == 1.2.0 + $ hub install humanseg_lite == 1.3.0 ``` diff --git a/modules/image/semantic_segmentation/humanseg_lite/README_en.md b/modules/image/semantic_segmentation/humanseg_lite/README_en.md index 4240a187..c6a41793 100644 --- a/modules/image/semantic_segmentation/humanseg_lite/README_en.md +++ b/modules/image/semantic_segmentation/humanseg_lite/README_en.md @@ -1,7 +1,7 @@ # humanseg_lite |Module Name |humanseg_lite| -| :--- | :---: | +| :--- | :---: | |Category |Image segmentation| |Network|shufflenet| |Dataset|Baidu self-built dataset| @@ -10,10 +10,10 @@ |Data indicators|-| |Latest update date|2021-02-26| -## I. Basic Information +## I. Basic Information - ### Application Effect Display - + - Sample results:

@@ -39,7 +39,7 @@ - ```shell $ hub install humanseg_lite ``` - + - In case of any problems during installation, please refer to:[Windows_Quickstart](../../../../docs/docs_en/get_start/windows_quickstart.md) | [Linux_Quickstart](../../../../docs/docs_en/get_start/linux_quickstart.md) | [Mac_Quickstart](../../../../docs/docs_en/get_start/mac_quickstart.md) @@ -49,7 +49,7 @@ - ``` hub run humanseg_lite --input_path "/PATH/TO/IMAGE" - + ``` - If you want to call the Hub module through the command line, please refer to: [PaddleHub Command Line Instruction](../../../../docs/docs_en/tutorial/cmd_usage.rst) @@ -122,9 +122,9 @@ - **Return** - * res (list\[dict\]): The list of recognition results, where each element is dict and each field is: + * res (list\[dict\]): The list of recognition results, where each element is dict and each field is: * save\_path (str, optional): Save path of the result. - * data (numpy.ndarray): The result of portrait segmentation. + * data (numpy.ndarray): The result of portrait segmentation. - ```python def video_stream_segment(self, @@ -230,6 +230,8 @@ cv2.imwrite("segment_human_lite.png", rgba) ``` +- ### Gradio APP support + Starting with PaddleHub 2.3.1, the Gradio APP for humanseg_lite is supported to be accessed in the browser using the link http://127.0.0.1:8866/gradio/humanseg_lite. ## V. Release Note @@ -238,7 +240,7 @@ First release - 1.1.0 - + Added video portrait segmentation interface Added video stream portrait segmentation interface @@ -251,6 +253,10 @@ Remove Fluid API +* 1.3.0 + + Add Gradio APP support. + ```shell - $ hub install humanseg_lite == 1.2.0 + $ hub install humanseg_lite == 1.3.0 ``` diff --git a/modules/image/semantic_segmentation/humanseg_lite/module.py b/modules/image/semantic_segmentation/humanseg_lite/module.py index 600d4c28..17a17159 100644 --- a/modules/image/semantic_segmentation/humanseg_lite/module.py +++ b/modules/image/semantic_segmentation/humanseg_lite/module.py @@ -12,32 +12,39 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import argparse import ast import os import os.path as osp -import argparse import cv2 import numpy as np -import paddle import paddle.jit import paddle.static -from paddle.inference import Config, create_predictor -from paddlehub.module.module import moduleinfo, runnable, serving - -from .processor import postprocess, base64_to_cv2, cv2_to_base64, check_dir -from .data_feed import reader, preprocess_v -from .optimal import postprocess_v, threshold_mask - - -@moduleinfo( - name="humanseg_lite", - type="CV/semantic_segmentation", - author="paddlepaddle", - author_email="", - summary="humanseg_lite is a semantic segmentation model.", - version="1.2.0") +from paddle.inference import Config +from paddle.inference import create_predictor + +from .data_feed import preprocess_v +from .data_feed import reader +from .optimal import postprocess_v +from .optimal import threshold_mask +from .processor import base64_to_cv2 +from .processor import check_dir +from .processor import cv2_to_base64 +from .processor import postprocess +from paddlehub.module.module import moduleinfo +from paddlehub.module.module import runnable +from paddlehub.module.module import serving + + +@moduleinfo(name="humanseg_lite", + type="CV/semantic_segmentation", + author="paddlepaddle", + author_email="", + summary="humanseg_lite is a semantic segmentation model.", + version="1.3.0") class ShufflenetHumanSeg: + def __init__(self): self.default_pretrained_model_path = os.path.join(self.directory, "humanseg_lite_inference", "model") self._set_config() @@ -46,8 +53,8 @@ class ShufflenetHumanSeg: """ predictor config setting """ - model = self.default_pretrained_model_path+'.pdmodel' - params = self.default_pretrained_model_path+'.pdiparams' + model = self.default_pretrained_model_path + '.pdmodel' + params = self.default_pretrained_model_path + '.pdiparams' cpu_config = Config(model, params) cpu_config.disable_glog_info() cpu_config.disable_gpu() @@ -64,7 +71,7 @@ class ShufflenetHumanSeg: gpu_config = Config(model, params) gpu_config.disable_glog_info() gpu_config.enable_use_gpu(memory_pool_init_size_mb=1000, device_id=0) - + if paddle.get_cudnn_version() == 8004: gpu_config.delete_pass('conv_elementwise_add_act_fuse_pass') gpu_config.delete_pass('conv_elementwise_add2_act_fuse_pass') @@ -134,13 +141,12 @@ class ShufflenetHumanSeg: output = np.expand_dims(output[:, 1, :, :], axis=1) # postprocess one by one for i in range(len(batch_data)): - out = postprocess( - data_out=output[i], - org_im=batch_data[i]['org_im'], - org_im_shape=batch_data[i]['org_im_shape'], - org_im_path=batch_data[i]['org_im_path'], - output_dir=output_dir, - visualization=visualization) + out = postprocess(data_out=output[i], + org_im=batch_data[i]['org_im'], + org_im_shape=batch_data[i]['org_im_shape'], + org_im_path=batch_data[i]['org_im_path'], + output_dir=output_dir, + visualization=visualization) res.append(out) return res @@ -327,11 +333,10 @@ class ShufflenetHumanSeg: """ Run as a command. """ - self.parser = argparse.ArgumentParser( - description="Run the {} module.".format(self.name), - prog='hub run {}'.format(self.name), - usage='%(prog)s', - add_help=True) + self.parser = argparse.ArgumentParser(description="Run the {} module.".format(self.name), + prog='hub run {}'.format(self.name), + usage='%(prog)s', + add_help=True) self.arg_input_group = self.parser.add_argument_group(title="Input options", description="Input data. Required") self.arg_config_group = self.parser.add_argument_group( @@ -339,12 +344,11 @@ class ShufflenetHumanSeg: self.add_module_config_arg() self.add_module_input_arg() args = self.parser.parse_args(argvs) - results = self.segment( - paths=[args.input_path], - batch_size=args.batch_size, - use_gpu=args.use_gpu, - output_dir=args.output_dir, - visualization=args.visualization) + results = self.segment(paths=[args.input_path], + batch_size=args.batch_size, + use_gpu=args.use_gpu, + output_dir=args.output_dir, + visualization=args.visualization) if args.save_dir is not None: check_dir(args.save_dir) self.save_inference_model(args.save_dir) @@ -355,14 +359,22 @@ class ShufflenetHumanSeg: """ Add the command config options. """ - self.arg_config_group.add_argument( - '--use_gpu', type=ast.literal_eval, default=False, help="whether use GPU or not") - self.arg_config_group.add_argument( - '--output_dir', type=str, default='humanseg_lite_output', help="The directory to save output images.") - self.arg_config_group.add_argument( - '--save_dir', type=str, default='humanseg_lite_model', help="The directory to save model.") - self.arg_config_group.add_argument( - '--visualization', type=ast.literal_eval, default=False, help="whether to save output as images.") + self.arg_config_group.add_argument('--use_gpu', + type=ast.literal_eval, + default=False, + help="whether use GPU or not") + self.arg_config_group.add_argument('--output_dir', + type=str, + default='humanseg_lite_output', + help="The directory to save output images.") + self.arg_config_group.add_argument('--save_dir', + type=str, + default='humanseg_lite_model', + help="The directory to save model.") + self.arg_config_group.add_argument('--visualization', + type=ast.literal_eval, + default=False, + help="whether to save output as images.") self.arg_config_group.add_argument('--batch_size', type=ast.literal_eval, default=1, help="batch size.") def add_module_input_arg(self): @@ -371,33 +383,20 @@ class ShufflenetHumanSeg: """ self.arg_input_group.add_argument('--input_path', type=str, help="path to image.") - -if __name__ == "__main__": - m = ShufflenetHumanSeg() - #shuffle.video_segment() - img = cv2.imread('photo.jpg') - # res = m.segment(images=[img], visualization=True) - # print(res[0]['data']) - # m.video_segment('') - cap_video = cv2.VideoCapture('video_test.mp4') - fps = cap_video.get(cv2.CAP_PROP_FPS) - save_path = 'result_frame.avi' - width = int(cap_video.get(cv2.CAP_PROP_FRAME_WIDTH)) - height = int(cap_video.get(cv2.CAP_PROP_FRAME_HEIGHT)) - cap_out = cv2.VideoWriter(save_path, cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'), fps, (width, height)) - prev_gray = None - prev_cfd = None - while cap_video.isOpened(): - ret, frame_org = cap_video.read() - if ret: - [img_matting, prev_gray, prev_cfd] = m.video_stream_segment( - frame_org=frame_org, frame_id=cap_video.get(1), prev_gray=prev_gray, prev_cfd=prev_cfd) - img_matting = np.repeat(img_matting[:, :, np.newaxis], 3, axis=2) - bg_im = np.ones_like(img_matting) * 255 - comb = (img_matting * frame_org + (1 - img_matting) * bg_im).astype(np.uint8) - cap_out.write(comb) - else: - break - - cap_video.release() - cap_out.release() + def create_gradio_app(self): + import gradio as gr + import tempfile + import os + from PIL import Image + + def inference(image, use_gpu=False): + with tempfile.TemporaryDirectory() as temp_dir: + self.segment(paths=[image], use_gpu=use_gpu, visualization=True, output_dir=temp_dir) + return Image.open(os.path.join(temp_dir, os.listdir(temp_dir)[0])) + + interface = gr.Interface( + inference, + [gr.inputs.Image(type="filepath"), gr.Checkbox(label='use_gpu')], + gr.outputs.Image(type="ndarray"), + title='humanseg_lite') + return interface diff --git a/modules/image/semantic_segmentation/humanseg_lite/optimal.py b/modules/image/semantic_segmentation/humanseg_lite/optimal.py index da7aa514..aeae7c50 100644 --- a/modules/image/semantic_segmentation/humanseg_lite/optimal.py +++ b/modules/image/semantic_segmentation/humanseg_lite/optimal.py @@ -32,8 +32,8 @@ def human_seg_tracking(pre_gray, cur_gray, prev_cfd, dl_weights, disflow): # 超出边界不跟踪 not_track = (cur_x < 0) + (cur_x >= w) + (cur_y < 0) + (cur_y >= h) flow_bw[~not_track] = flow_bw[cur_y[~not_track], cur_x[~not_track]] - not_track += ( - np.square(flow_fw[:, :, 0] + flow_bw[:, :, 0]) + np.square(flow_fw[:, :, 1] + flow_bw[:, :, 1])) >= check_thres + not_track += (np.square(flow_fw[:, :, 0] + flow_bw[:, :, 0]) + + np.square(flow_fw[:, :, 1] + flow_bw[:, :, 1])) >= check_thres track_cfd[cur_y[~not_track], cur_x[~not_track]] = prev_cfd[~not_track] is_track[cur_y[~not_track], cur_x[~not_track]] = 1 diff --git a/modules/image/semantic_segmentation/humanseg_lite/processor.py b/modules/image/semantic_segmentation/humanseg_lite/processor.py index 9cd53a84..a3e83622 100644 --- a/modules/image/semantic_segmentation/humanseg_lite/processor.py +++ b/modules/image/semantic_segmentation/humanseg_lite/processor.py @@ -1,7 +1,7 @@ # -*- coding:utf-8 -*- +import base64 import os import time -import base64 import cv2 import numpy as np diff --git a/modules/image/semantic_segmentation/humanseg_lite/test.py b/modules/image/semantic_segmentation/humanseg_lite/test.py index df433469..b864da71 100644 --- a/modules/image/semantic_segmentation/humanseg_lite/test.py +++ b/modules/image/semantic_segmentation/humanseg_lite/test.py @@ -3,15 +3,16 @@ import shutil import unittest import cv2 -import requests import numpy as np -import paddlehub as hub +import requests +import paddlehub as hub os.environ['CUDA_VISIBLE_DEVICES'] = '0' class TestHubModule(unittest.TestCase): + @classmethod def setUpClass(cls) -> None: img_url = 'https://unsplash.com/photos/pg_WCHWSdT8/download?ixid=MnwxMjA3fDB8MXxhbGx8fHx8fHx8fHwxNjYyNDM2ODI4&force=true&w=640' @@ -23,8 +24,7 @@ class TestHubModule(unittest.TestCase): f.write(response.content) fourcc = cv2.VideoWriter_fourcc('M', 'J', 'P', 'G') img = cv2.imread('tests/test.jpg') - video = cv2.VideoWriter('tests/test.avi', fourcc, - 20.0, tuple(img.shape[:2])) + video = cv2.VideoWriter('tests/test.avi', fourcc, 20.0, tuple(img.shape[:2])) for i in range(40): video.write(img) video.release() @@ -38,101 +38,65 @@ class TestHubModule(unittest.TestCase): shutil.rmtree('humanseg_lite_video_result') def test_segment1(self): - results = self.module.segment( - paths=['tests/test.jpg'], - use_gpu=False, - visualization=False - ) + results = self.module.segment(paths=['tests/test.jpg'], use_gpu=False, visualization=False) self.assertIsInstance(results[0]['data'], np.ndarray) def test_segment2(self): - results = self.module.segment( - images=[cv2.imread('tests/test.jpg')], - use_gpu=False, - visualization=False - ) + results = self.module.segment(images=[cv2.imread('tests/test.jpg')], use_gpu=False, visualization=False) self.assertIsInstance(results[0]['data'], np.ndarray) def test_segment3(self): - results = self.module.segment( - images=[cv2.imread('tests/test.jpg')], - use_gpu=False, - visualization=True - ) + results = self.module.segment(images=[cv2.imread('tests/test.jpg')], use_gpu=False, visualization=True) self.assertIsInstance(results[0]['data'], np.ndarray) def test_segment4(self): - results = self.module.segment( - images=[cv2.imread('tests/test.jpg')], - use_gpu=True, - visualization=False - ) + results = self.module.segment(images=[cv2.imread('tests/test.jpg')], use_gpu=True, visualization=False) self.assertIsInstance(results[0]['data'], np.ndarray) def test_segment5(self): - self.assertRaises( - AssertionError, - self.module.segment, - paths=['no.jpg'] - ) + self.assertRaises(AssertionError, self.module.segment, paths=['no.jpg']) def test_segment6(self): - self.assertRaises( - AttributeError, - self.module.segment, - images=['test.jpg'] - ) + self.assertRaises(AttributeError, self.module.segment, images=['test.jpg']) def test_video_stream_segment1(self): - img_matting, cur_gray, optflow_map = self.module.video_stream_segment( - frame_org=cv2.imread('tests/test.jpg'), - frame_id=1, - prev_gray=None, - prev_cfd=None, - use_gpu=False - ) + img_matting, cur_gray, optflow_map = self.module.video_stream_segment(frame_org=cv2.imread('tests/test.jpg'), + frame_id=1, + prev_gray=None, + prev_cfd=None, + use_gpu=False) self.assertIsInstance(img_matting, np.ndarray) self.assertIsInstance(cur_gray, np.ndarray) self.assertIsInstance(optflow_map, np.ndarray) - img_matting, cur_gray, optflow_map = self.module.video_stream_segment( - frame_org=cv2.imread('tests/test.jpg'), - frame_id=2, - prev_gray=cur_gray, - prev_cfd=optflow_map, - use_gpu=False - ) + img_matting, cur_gray, optflow_map = self.module.video_stream_segment(frame_org=cv2.imread('tests/test.jpg'), + frame_id=2, + prev_gray=cur_gray, + prev_cfd=optflow_map, + use_gpu=False) self.assertIsInstance(img_matting, np.ndarray) self.assertIsInstance(cur_gray, np.ndarray) self.assertIsInstance(optflow_map, np.ndarray) def test_video_stream_segment2(self): - img_matting, cur_gray, optflow_map = self.module.video_stream_segment( - frame_org=cv2.imread('tests/test.jpg'), - frame_id=1, - prev_gray=None, - prev_cfd=None, - use_gpu=True - ) + img_matting, cur_gray, optflow_map = self.module.video_stream_segment(frame_org=cv2.imread('tests/test.jpg'), + frame_id=1, + prev_gray=None, + prev_cfd=None, + use_gpu=True) self.assertIsInstance(img_matting, np.ndarray) self.assertIsInstance(cur_gray, np.ndarray) self.assertIsInstance(optflow_map, np.ndarray) - img_matting, cur_gray, optflow_map = self.module.video_stream_segment( - frame_org=cv2.imread('tests/test.jpg'), - frame_id=2, - prev_gray=cur_gray, - prev_cfd=optflow_map, - use_gpu=True - ) + img_matting, cur_gray, optflow_map = self.module.video_stream_segment(frame_org=cv2.imread('tests/test.jpg'), + frame_id=2, + prev_gray=cur_gray, + prev_cfd=optflow_map, + use_gpu=True) self.assertIsInstance(img_matting, np.ndarray) self.assertIsInstance(cur_gray, np.ndarray) self.assertIsInstance(optflow_map, np.ndarray) def test_video_segment1(self): - self.module.video_segment( - video_path="tests/test.avi", - use_gpu=False, - save_dir='humanseg_lite_video_result' - ) + self.module.video_segment(video_path="tests/test.avi", use_gpu=False, save_dir='humanseg_lite_video_result') def test_save_inference_model(self): self.module.save_inference_model('./inference/model') -- GitLab