diff --git a/modules/image/instance_segmentation/solov2/README.md b/modules/image/instance_segmentation/solov2/README.md index a9ca8e14e6587983a079d0f2f6a3069717fea2a5..16e1b3d24a0820fab7cc60a1cca66f67d0f93a17 100644 --- a/modules/image/instance_segmentation/solov2/README.md +++ b/modules/image/instance_segmentation/solov2/README.md @@ -78,7 +78,7 @@ - res (dict): 识别结果，关键字有 'segm', 'label', 'score'对应的取值为： - segm (np.ndarray): 实例分割结果,取值为0或1。0表示背景，1为实例； - label (list): 实例分割结果类别id； - - score (list):实例分割结果类别得分；s + - score (list):实例分割结果类别得分； ## 四、服务部署 @@ -147,8 +147,10 @@ 初始发布 - * ```shell - $ hub install hand_pose_localization==1.0.0 - ``` +* 1.1.0 - + 适配 PaddlePaddle 2.2.0+ + + * ```shell + $ hub install hand_pose_localization==1.1.0 + ``` \ No newline at end of file diff --git a/modules/image/instance_segmentation/solov2/data_feed.py b/modules/image/instance_segmentation/solov2/data_feed.py index 7d34afe8567235241163c81e416b3c8ee1c80445..1baaefafd450fd08c13bd4021f4c165a4f41b9c1 100644 --- a/modules/image/instance_segmentation/solov2/data_feed.py +++ b/modules/image/instance_segmentation/solov2/data_feed.py @@ -3,8 +3,8 @@ import base64 import cv2 import numpy as np +from paddle.inference import Config, create_predictor, PrecisionType from PIL import Image, ImageDraw -import paddle.fluid as fluid def create_inputs(im, im_info): @@ -19,11 +19,14 @@ def create_inputs(im, im_info): inputs['image'] = im origin_shape = list(im_info['origin_shape']) resize_shape = list(im_info['resize_shape']) - pad_shape = list(im_info['pad_shape']) if im_info['pad_shape'] is not None else list(im_info['resize_shape']) + pad_shape = list(im_info['pad_shape']) if im_info[ + 'pad_shape'] is not None else list(im_info['resize_shape']) scale_x, scale_y = im_info['scale'] scale = scale_x im_info = np.array([resize_shape + [scale]]).astype('float32') inputs['im_info'] = im_info + inputs['scale_factor'] = np.array([scale_x, scale_x]).astype('float32').reshape(-1, 2) + inputs['im_shape'] = np.array(resize_shape).astype('float32').reshape(-1, 2) return inputs @@ -42,28 +45,38 @@ def visualize_box_mask(im, results, labels=None, mask_resolution=14, threshold=0 im (PIL.Image.Image): visualized image """ if not labels: - labels = [ - 'background', 'person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat', - 'traffic light', 'fire', 'hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', - 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie', - 'suitcase', 'frisbee', 'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove', - 'skateboard', 'surfboard', 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', - 'banana', 'apple', 'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair', - 'couch', 'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote', 'keyboard', - 'cell phone', 'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors', - 'teddy bear', 'hair drier', 'toothbrush' - ] + labels = ['background', 'person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', + 'train', 'truck', 'boat', 'traffic light', 'fire', 'hydrant', 'stop sign', 'parking meter', + 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', + 'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee', 'skis', 'snowboard', 'sports ball', + 'kite', 'baseball bat', 'baseball glove', 'skateboard', 'surfboard', 'tennis racket', 'bottle', + 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple', 'sandwich', 'orange', + 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch', 'potted plant', 'bed', + 'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote', 'keyboard', 'cell phone', 'microwave', + 'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors', 'teddy bear', + 'hair drier', 'toothbrush'] if isinstance(im, str): im = Image.open(im).convert('RGB') else: im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) im = Image.fromarray(im) if 'masks' in results and 'boxes' in results: - im = draw_mask(im, results['boxes'], results['masks'], labels, resolution=mask_resolution) + im = draw_mask( + im, + results['boxes'], + results['masks'], + labels, + resolution=mask_resolution) if 'boxes' in results: im = draw_box(im, results['boxes'], labels) if 'segm' in results: - im = draw_segm(im, results['segm'], results['label'], results['score'], labels, threshold=threshold) + im = draw_segm( + im, + results['segm'], + results['label'], + results['score'], + labels, + threshold=threshold) return im @@ -152,7 +165,8 @@ def draw_mask(im, np_boxes, np_masks, labels, resolution=14, threshold=0.5): y0 = min(max(ymin, 0), im_h) y1 = min(max(ymax + 1, 0), im_h) im_mask = np.zeros((im_h, im_w), dtype=np.uint8) - im_mask[y0:y1, x0:x1] = resized_mask[(y0 - ymin):(y1 - ymin), (x0 - xmin):(x1 - xmin)] + im_mask[y0:y1, x0:x1] = resized_mask[(y0 - ymin):(y1 - ymin), ( + x0 - xmin):(x1 - xmin)] if clsid not in clsid2color: clsid2color[clsid] = color_list[clsid] color_mask = clsid2color[clsid] @@ -190,19 +204,28 @@ def draw_box(im, np_boxes, labels): color = tuple(clsid2color[clsid]) # draw bbox - draw.line([(xmin, ymin), (xmin, ymax), (xmax, ymax), (xmax, ymin), (xmin, ymin)], - width=draw_thickness, - fill=color) + draw.line( + [(xmin, ymin), (xmin, ymax), (xmax, ymax), (xmax, ymin), + (xmin, ymin)], + width=draw_thickness, + fill=color) # draw label text = "{} {:.4f}".format(labels[clsid], score) tw, th = draw.textsize(text) - draw.rectangle([(xmin + 1, ymin - th), (xmin + tw + 1, ymin)], fill=color) + draw.rectangle( + [(xmin + 1, ymin - th), (xmin + tw + 1, ymin)], fill=color) draw.text((xmin + 1, ymin - th), text, fill=(255, 255, 255)) return im -def draw_segm(im, np_segms, np_label, np_score, labels, threshold=0.5, alpha=0.7): +def draw_segm(im, + np_segms, + np_label, + np_score, + labels, + threshold=0.5, + alpha=0.7): """ Draw segmentation on image. """ @@ -231,17 +254,28 @@ def draw_segm(im, np_segms, np_label, np_score, labels, threshold=0.5, alpha=0.7 sum_y = np.sum(mask, axis=1) y = np.where(sum_y > 0.5)[0] x0, x1, y0, y1 = x[0], x[-1], y[0], y[-1] - cv2.rectangle(im, (x0, y0), (x1, y1), tuple(color_mask.astype('int32').tolist()), 1) + cv2.rectangle(im, (x0, y0), (x1, y1), + tuple(color_mask.astype('int32').tolist()), 1) bbox_text = '%s %.2f' % (labels[clsid], score) t_size = cv2.getTextSize(bbox_text, 0, 0.3, thickness=1)[0] - cv2.rectangle(im, (x0, y0), (x0 + t_size[0], y0 - t_size[1] - 3), tuple(color_mask.astype('int32').tolist()), - -1) - cv2.putText(im, bbox_text, (x0, y0 - 2), cv2.FONT_HERSHEY_SIMPLEX, 0.3, (0, 0, 0), 1, lineType=cv2.LINE_AA) - + cv2.rectangle(im, (x0, y0), (x0 + t_size[0], y0 - t_size[1] - 3), + tuple(color_mask.astype('int32').tolist()), -1) + cv2.putText( + im, + bbox_text, (x0, y0 - 2), + cv2.FONT_HERSHEY_SIMPLEX, + 0.3, (0, 0, 0), + 1, + lineType=cv2.LINE_AA) + return Image.fromarray(im.astype('uint8')) -def load_predictor(model_dir, run_mode='fluid', batch_size=1, use_gpu=False, min_subgraph_size=3): +def load_predictor(model_dir, + run_mode='paddle', + batch_size=1, + use_gpu=False, + min_subgraph_size=3): """set AnalysisConfig, generate AnalysisPredictor Args: model_dir (str): root path of __model__ and __params__ @@ -251,17 +285,19 @@ def load_predictor(model_dir, run_mode='fluid', batch_size=1, use_gpu=False, min Raises: ValueError: predict by TensorRT need use_gpu == True. """ - if not use_gpu and not run_mode == 'fluid': - raise ValueError("Predict by TensorRT mode: {}, expect use_gpu==True, but use_gpu == {}".format( - run_mode, use_gpu)) + if not use_gpu and not run_mode == 'paddle': + raise ValueError( + "Predict by TensorRT mode: {}, expect use_gpu==True, but use_gpu == {}" + .format(run_mode, use_gpu)) if run_mode == 'trt_int8': - raise ValueError("TensorRT int8 mode is not supported now, " "please use trt_fp32 or trt_fp16 instead.") + raise ValueError("TensorRT int8 mode is not supported now, " + "please use trt_fp32 or trt_fp16 instead.") precision_map = { - 'trt_int8': fluid.core.AnalysisConfig.Precision.Int8, - 'trt_fp32': fluid.core.AnalysisConfig.Precision.Float32, - 'trt_fp16': fluid.core.AnalysisConfig.Precision.Half + 'trt_int8': PrecisionType.Int8, + 'trt_fp32': PrecisionType.Float32, + 'trt_fp16': PrecisionType.Half } - config = fluid.core.AnalysisConfig(os.path.join(model_dir, '__model__'), os.path.join(model_dir, '__params__')) + config = Config(model_dir+'.pdmodel', model_dir+'.pdiparams') if use_gpu: # initial GPU memory(M), device ID config.enable_use_gpu(100, 0) @@ -285,7 +321,7 @@ def load_predictor(model_dir, run_mode='fluid', batch_size=1, use_gpu=False, min config.enable_memory_optim() # disable feed, fetch OP, needed by zero_copy_run config.switch_use_feed_fetch_ops(False) - predictor = fluid.core.create_paddle_predictor(config) + predictor = create_predictor(config) return predictor diff --git a/modules/image/instance_segmentation/solov2/example.png b/modules/image/instance_segmentation/solov2/example.png deleted file mode 100644 index 4ece0a2df78452484829ce4e0fafd02341c5a3e5..0000000000000000000000000000000000000000 Binary files a/modules/image/instance_segmentation/solov2/example.png and /dev/null differ diff --git a/modules/image/instance_segmentation/solov2/module.py b/modules/image/instance_segmentation/solov2/module.py index 520300319207b77bdffb96797bb0054bdac474d0..eaf8616bc1576dfa6682a542fa5762bb53211dc8 100644 --- a/modules/image/instance_segmentation/solov2/module.py +++ b/modules/image/instance_segmentation/solov2/module.py @@ -11,13 +11,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + import os import time import base64 from functools import reduce from typing import Union -import cv2 import numpy as np from paddlehub.module.module import moduleinfo, serving @@ -25,7 +25,7 @@ import solov2.processor as P import solov2.data_feed as D -class Detector(object): +class Detector: """ Args: min_subgraph_size (int): number of tensorRT graphs. @@ -33,23 +33,26 @@ class Detector(object): threshold (float): threshold to reserve the result for output. """ - def __init__(self, min_subgraph_size: int = 60, use_gpu=False, threshold: float = 0.5): + def __init__(self, + min_subgraph_size: int = 60, + use_gpu=False): - model_dir = os.path.join(self.directory, 'solov2_r50_fpn_1x') - self.predictor = D.load_predictor(model_dir, min_subgraph_size=min_subgraph_size, use_gpu=use_gpu) - self.compose = [ - P.Resize(max_size=1333), - P.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), - P.Permute(), - P.PadStride(stride=32) - ] + self.default_pretrained_model_path = os.path.join(self.directory, 'solov2_r50_fpn_1x', 'model') + self.predictor = D.load_predictor( + self.default_pretrained_model_path, + min_subgraph_size=min_subgraph_size, + use_gpu=use_gpu) + self.compose = [P.Resize(max_size=1333), + P.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), + P.Permute(), + P.PadStride(stride=32)] def transform(self, im: Union[str, np.ndarray]): im, im_info = P.preprocess(im, self.compose) inputs = D.create_inputs(im, im_info) return inputs, im_info - def postprocess(self, np_boxes: np.ndarray, np_masks: np.ndarray, im_info: dict, threshold: float = 0.5): + def postprocess(self, np_boxes: np.ndarray, np_masks: np.ndarray, threshold: float = 0.5): # postprocess output of predictor results = {} expect_boxes = (np_boxes[:, 1] > threshold) & (np_boxes[:, 0] > -1) @@ -57,14 +60,17 @@ class Detector(object): for box in np_boxes: print('class_id:{:d}, confidence:{:.4f},' 'left_top:[{:.2f},{:.2f}],' - ' right_bottom:[{:.2f},{:.2f}]'.format(int(box[0]), box[1], box[2], box[3], box[4], box[5])) + ' right_bottom:[{:.2f},{:.2f}]'.format( + int(box[0]), box[1], box[2], box[3], box[4], box[5])) results['boxes'] = np_boxes if np_masks is not None: np_masks = np_masks[expect_boxes, :, :, :] results['masks'] = np_masks return results - def predict(self, image: Union[str, np.ndarray], threshold: float = 0.5): + def predict(self, + image: Union[str, np.ndarray], + threshold: float = 0.5): ''' Args: image (str/np.ndarray): path of image/ np.ndarray read by cv2 @@ -80,12 +86,12 @@ class Detector(object): input_names = self.predictor.get_input_names() for i in range(len(input_names)): - input_tensor = self.predictor.get_input_tensor(input_names[i]) + input_tensor = self.predictor.get_input_handle(input_names[i]) input_tensor.copy_from_cpu(inputs[input_names[i]]) - self.predictor.zero_copy_run() + self.predictor.run() output_names = self.predictor.get_output_names() - boxes_tensor = self.predictor.get_output_tensor(output_names[0]) + boxes_tensor = self.predictor.get_output_handle(output_names[0]) np_boxes = boxes_tensor.copy_to_cpu() # do not perform postprocess in benchmark mode results = [] @@ -103,16 +109,18 @@ class Detector(object): author="paddlepaddle", author_email="", summary="solov2 is a detection model, this module is trained with COCO dataset.", - version="1.0.0") + version="1.1.0") class DetectorSOLOv2(Detector): """ Args: use_gpu (bool): whether use gpu threshold (float): threshold to reserve the result for output. """ + def __init__(self, + use_gpu: bool = False): + super(DetectorSOLOv2, self).__init__( + use_gpu=use_gpu) - def __init__(self, use_gpu: bool = False, threshold: float = 0.5): - super(DetectorSOLOv2, self).__init__(use_gpu=use_gpu, threshold=threshold) def predict(self, image: Union[str, np.ndarray], @@ -125,7 +133,7 @@ class DetectorSOLOv2(Detector): threshold (float): threshold of predicted box' score visualization (bool): Whether to save visualization result. save_dir (str): save path. - + ''' inputs, im_info = self.transform(image) @@ -133,20 +141,23 @@ class DetectorSOLOv2(Detector): input_names = self.predictor.get_input_names() for i in range(len(input_names)): - input_tensor = self.predictor.get_input_tensor(input_names[i]) + input_tensor = self.predictor.get_input_handle(input_names[i]) input_tensor.copy_from_cpu(inputs[input_names[i]]) - self.predictor.zero_copy_run() + self.predictor.run() output_names = self.predictor.get_output_names() - np_label = self.predictor.get_output_tensor(output_names[0]).copy_to_cpu() - np_score = self.predictor.get_output_tensor(output_names[1]).copy_to_cpu() - np_segms = self.predictor.get_output_tensor(output_names[2]).copy_to_cpu() + np_label = self.predictor.get_output_handle(output_names[ + 1]).copy_to_cpu() + np_score = self.predictor.get_output_handle(output_names[ + 2]).copy_to_cpu() + np_segms = self.predictor.get_output_handle(output_names[ + 3]).copy_to_cpu() output = dict(segm=np_segms, label=np_label, score=np_score) - + if visualization: if not os.path.exists(save_dir): os.makedirs(save_dir) - image = D.visualize_box_mask(im=image, results=output) + image = D.visualize_box_mask(im=image, results=output, threshold=threshold) name = str(time.time()) + '.png' save_path = os.path.join(save_dir, name) image.save(save_path) @@ -163,4 +174,4 @@ class DetectorSOLOv2(Detector): final['segm'] = base64.b64encode(results['segm']).decode('utf8') final['label'] = base64.b64encode(results['label']).decode('utf8') final['score'] = base64.b64encode(results['score']).decode('utf8') - return final + return final \ No newline at end of file diff --git a/modules/image/instance_segmentation/solov2/processor.py b/modules/image/instance_segmentation/solov2/processor.py index 0de87f5a6a71d1e46fb5e67ec092bb2af70516e0..a5e4c9d85115639ebebd1547f6f399d9649ca3be 100644 --- a/modules/image/instance_segmentation/solov2/processor.py +++ b/modules/image/instance_segmentation/solov2/processor.py @@ -78,13 +78,20 @@ class Resize(object): im_channel = im.shape[2] im_scale_x, im_scale_y = self.generate_scale(im) if self.use_cv2: - im = cv2.resize(im, None, None, fx=im_scale_x, fy=im_scale_y, interpolation=self.interp) + im = cv2.resize( + im, + None, + None, + fx=im_scale_x, + fy=im_scale_y, + interpolation=self.interp) else: resize_w = int(im_scale_x * float(im.shape[1])) resize_h = int(im_scale_y * float(im.shape[0])) if self.max_size != 0: - raise TypeError('If you set max_size to cap the maximum size of image,' - 'please set use_cv2 to True to resize the image.') + raise TypeError( + 'If you set max_size to cap the maximum size of image,' + 'please set use_cv2 to True to resize the image.') im = im.astype('uint8') im = Image.fromarray(im) im = im.resize((int(resize_w), int(resize_h)), self.interp) @@ -92,7 +99,8 @@ class Resize(object): # padding im when image_shape fixed by infer_cfg.yml if self.max_size != 0 and self.image_shape is not None: - padding_im = np.zeros((self.max_size, self.max_size, im_channel), dtype=np.float32) + padding_im = np.zeros( + (self.max_size, self.max_size, im_channel), dtype=np.float32) im_h, im_w = im.shape[:2] padding_im[:im_h, :im_w, :] = im im = padding_im @@ -232,4 +240,4 @@ def preprocess(im, preprocess_ops): for operator in preprocess_ops: im, im_info = operator(im, im_info) im = np.array((im, )).astype('float32') - return im, im_info + return im, im_info \ No newline at end of file diff --git a/modules/image/instance_segmentation/solov2/test.py b/modules/image/instance_segmentation/solov2/test.py new file mode 100644 index 0000000000000000000000000000000000000000..b3dfd416eb542fb49be316d42a3cb80876590baf --- /dev/null +++ b/modules/image/instance_segmentation/solov2/test.py @@ -0,0 +1,96 @@ +import os +import shutil +import unittest + +import cv2 +import requests +import numpy as np +import paddlehub as hub + + +os.environ['CUDA_VISIBLE_DEVICES'] = '0' + + +class TestHubModule(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + img_url = 'https://ai-studio-static-online.cdn.bcebos.com/7799a8ccc5f6471b9d56fb6eff94f82a08b70ca2c7594d3f99877e366c0a2619' + if not os.path.exists('tests'): + os.makedirs('tests') + response = requests.get(img_url) + assert response.status_code == 200, 'Network Error.' + with open('tests/test.jpg', 'wb') as f: + f.write(response.content) + cls.module = hub.Module(name="solov2") + + @classmethod + def tearDownClass(cls) -> None: + shutil.rmtree('tests') + shutil.rmtree('inference') + shutil.rmtree('solov2_result') + + def test_predict1(self): + results = self.module.predict( + image='tests/test.jpg', + visualization=False + ) + segm = results['segm'] + label = results['label'] + score = results['score'] + self.assertIsInstance(segm, np.ndarray) + self.assertIsInstance(label, np.ndarray) + self.assertIsInstance(score, np.ndarray) + + def test_predict2(self): + results = self.module.predict( + image=cv2.imread('tests/test.jpg'), + visualization=False + ) + segm = results['segm'] + label = results['label'] + score = results['score'] + self.assertIsInstance(segm, np.ndarray) + self.assertIsInstance(label, np.ndarray) + self.assertIsInstance(score, np.ndarray) + + def test_predict3(self): + results = self.module.predict( + image=cv2.imread('tests/test.jpg'), + visualization=True + ) + segm = results['segm'] + label = results['label'] + score = results['score'] + self.assertIsInstance(segm, np.ndarray) + self.assertIsInstance(label, np.ndarray) + self.assertIsInstance(score, np.ndarray) + + def test_predict4(self): + module = hub.Module(name="solov2", use_gpu=True) + results = module.predict( + image=cv2.imread('tests/test.jpg'), + visualization=True + ) + segm = results['segm'] + label = results['label'] + score = results['score'] + self.assertIsInstance(segm, np.ndarray) + self.assertIsInstance(label, np.ndarray) + self.assertIsInstance(score, np.ndarray) + + def test_predict5(self): + self.assertRaises( + FileNotFoundError, + self.module.predict, + image='no.jpg' + ) + + def test_save_inference_model(self): + self.module.save_inference_model('./inference/model') + + self.assertTrue(os.path.exists('./inference/model.pdmodel')) + self.assertTrue(os.path.exists('./inference/model.pdiparams')) + + +if __name__ == "__main__": + unittest.main()