Add PP-Matting (#5563)

modelcenter/PP-Matting/APP/app.py

+import codecs
+import os
+import sys
+import time
+import zipfile
+
+import gradio as gr
+import numpy as np
+import cv2
+import requests
+import yaml
+from paddle.inference import Config as PredictConfig
+from paddle.inference import create_predictor
+
+lasttime = time.time()
+FLUSH_INTERVAL = 0.1
+
+
+def progress(str, end=False):
+    global lasttime
+    if end:
+        str += "\n"
+        lasttime = 0
+    if time.time() - lasttime >= FLUSH_INTERVAL:
+        sys.stdout.write("\r%s" % str)
+        lasttime = time.time()
+        sys.stdout.flush()
+
+
+def _download_file(url, savepath, print_progress=True):
+    if print_progress:
+        print("Connecting to {}".format(url))
+    r = requests.get(url, stream=True, timeout=15)
+    total_length = r.headers.get('content-length')
+
+    if total_length is None:
+        with open(savepath, 'wb') as f:
+            shutil.copyfileobj(r.raw, f)
+    else:
+        with open(savepath, 'wb') as f:
+            dl = 0
+            total_length = int(total_length)
+            starttime = time.time()
+            if print_progress:
+                print("Downloading %s" % os.path.basename(savepath))
+            for data in r.iter_content(chunk_size=4096):
+                dl += len(data)
+                f.write(data)
+                if print_progress:
+                    done = int(50 * dl / total_length)
+                    progress("[%-50s] %.2f%%" %
+                             ('=' * done, float(100 * dl) / total_length))
+        if print_progress:
+            progress("[%-50s] %.2f%%" % ('=' * 50, 100), end=True)
+
+
+def uncompress(path):
+    files = zipfile.ZipFile(path, 'r')
+    filelist = files.namelist()
+    rootpath = filelist[0]
+    for file in filelist:
+        files.extract(file, './')
+
+
+class DeployConfig:
+    def __init__(self, path):
+        with codecs.open(path, 'r', 'utf-8') as file:
+            self.dic = yaml.load(file, Loader=yaml.FullLoader)
+        self._dir = os.path.dirname(path)
+
+    @property
+    def model(self):
+        return os.path.join(self._dir, self.dic['Deploy']['model'])
+
+    @property
+    def params(self):
+        return os.path.join(self._dir, self.dic['Deploy']['params'])
+
+
+class Predictor:
+    def __init__(self, cfg):
+        """
+        Prepare for prediction.
+        The usage and docs of paddle inference, please refer to
+        https://paddleinference.paddlepaddle.org.cn/product_introduction/summary.html
+        """
+        self.cfg = DeployConfig(cfg)
+
+        self._init_base_config()
+
+        self._init_cpu_config()
+
+        self.predictor = create_predictor(self.pred_cfg)
+
+    def _init_base_config(self):
+        self.pred_cfg = PredictConfig(self.cfg.model, self.cfg.params)
+        self.pred_cfg.enable_memory_optim()
+        self.pred_cfg.switch_ir_optim(True)
+
+    def _init_cpu_config(self):
+        """
+        Init the config for x86 cpu.
+        """
+        self.pred_cfg.disable_gpu()
+        self.pred_cfg.set_cpu_math_library_num_threads(10)
+
+    def _preprocess(self, img):
+        # resize short edge to 512.
+        h, w = img.shape[:2]
+        short_edge = min(h, w)
+        scale = 512 / short_edge
+        h_resize = int(round(h * scale)) // 32 * 32
+        w_resize = int(round(w * scale)) // 32 * 32
+        img = cv2.resize(img, (w_resize, h_resize))
+        img = (img / 255 - 0.5) / 0.5
+        img = np.transpose(img, [2, 0, 1])[np.newaxis, :]
+        return img
+
+    def run(self, img):
+        input_names = self.predictor.get_input_names()
+        input_handle = {}
+
+        for i in range(len(input_names)):
+            input_handle[input_names[i]] = self.predictor.get_input_handle(
+                input_names[i])
+        output_names = self.predictor.get_output_names()
+        output_handle = self.predictor.get_output_handle(output_names[0])
+
+        img_inputs = img.astype('float32')
+        ori_h, ori_w = img_inputs.shape[:2]
+        img_inputs = self._preprocess(img=img_inputs)
+        input_handle['img'].copy_from_cpu(img_inputs)
+
+        self.predictor.run()
+
+        results = output_handle.copy_to_cpu()
+        alpha = results.squeeze()
+        alpha = cv2.resize(alpha, (ori_w, ori_h))
+        alpha = (alpha * 255).astype('uint8')
+
+        return alpha
+
+
+def model_inference(image):
+    # Download inference model
+    url = 'https://paddleseg.bj.bcebos.com/matting/models/deploy/ppmatting-hrnet_w18-human_512.zip'
+    savepath = './ppmatting-hrnet_w18-human_512.zip'
+    if not os.path.exists('./ppmatting-hrnet_w18-human_512'):
+        _download_file(url=url, savepath=savepath)
+        uncompress(savepath)
+
+    # Inference
+    predictor = Predictor(cfg='./ppmatting-hrnet_w18-human_512/deploy.yaml')
+    alpha = predictor.run(image)
+
+    return alpha
+
+
+def clear_all():
+    return None, None
+
+
+with gr.Blocks() as demo:
+    gr.Markdown("Objective Detection")
+
+    with gr.Column(scale=1, min_width=100):
+
+        img_in = gr.Image(
+            value="https://paddleseg.bj.bcebos.com/matting/demo/human.jpg",
+            label="Input")
+
+        with gr.Row():
+            btn1 = gr.Button("Clear")
+            btn2 = gr.Button("Submit")
+
+        img_out = gr.Image(label="Output").style(height=200)
+
+    btn2.click(fn=model_inference, inputs=img_in, outputs=[img_out])
+    btn1.click(fn=clear_all, inputs=None, outputs=[img_in, img_out])
+    gr.Button.style(1)
+
+demo.launch(share=True)

modelcenter/PP-Matting/APP/app.yml

+【PP-Matting-App-YAML】
+
+APP_Info:
+    title: PP-Matting-App
+    colorFrom: blue
+    colorTo: yellow
+    sdk: gradio
+    sdk_version: 3.4.1
+    app_file: app.py
+    license: apache-2.0
+    device: cpu
\ No newline at end of file

modelcenter/PP-Matting/APP/requirement.txt

+gradio
+paddlepaddle
+opencv-python
+pyyaml >= 5.1
\ No newline at end of file

modelcenter/PP-Matting/benchmark_cn.md

+# 模型列表
+
+## 1. 训练Benchmark
+
+### 1.1 软硬件环境
+
+* PP-Matting训练过程中使用单卡GPU，batch size为4。
+
+### 1.2 数据集
+
+* 通用目标抠图数据集为Compositon-1k或Distinctions-646（使用该两者数据集需向作者进行申请），使用COCO2017和Pascal VOC 2012作为背景数据集。
+* 人像抠图使用内部数据。
+
+### 1.3 指标
+|模型名称 | 模型简介 | 输入尺寸 |
+|---|---|---|
+|ppmatting_hrnet_w48 | 通用目标抠图 | 512 |
+|ppmatting_hrnet_w18 | 人像抠图 | 512 |
+
+## 2. 推理Benchmark
+
+### 2.1 软硬件环境
+
+* 模型推理速度测试采用单卡V100，batch size=1进行测试，使用CUDA 10.2, CUDNN 7.6.5, PaddlePaddle-gpu 2.3.2。
+
+### 2.2 数据集
+* 通用目标抠图：Compositon-1k或Distinctions-646中的测试集部分。
+* 人像抠图：PPM-100和AIM-500中的人像部分，共195张, 记为PPM-AIM-195。
+
+### 2.3 指标
+| 模型 | 数据集 | SAD | MSE | Grad | Conn |Params(M) | FLOPs(G) | FPS |
+| - | - | -| - | - | - | - | -| - | - |
+| ppmatting_hrnet_w48 | Composition-1k | 46.22 | 0.005 | 22.69 | 45.40 | 86.3 | 165.4 | 24.4 |
+| ppmatting_hrnet_w48 | Distinctions-646 | 40.69 | 0.009 | 43.91 |40.56 | 86.3 | 165.4 | 24.4 |
+| ppmatting_hrnet_w18 | PPM-AIM-195 | 31.56|0.0022|31.80|30.13| 24.5 | 91.28 | 28.9 |
+
+## 3. 相关使用说明
+1. https://github.com/PaddlePaddle/PaddleSeg/tree/develop/Matting

modelcenter/PP-Matting/download_cn.md

+# 模型列表
+
+## 通用目标抠图
+|模型名称 | 数据集 | 模型简介 |  推理模型大小 | 下载地址 |
+|---|---|---|---|---|
+|ppmatting_hrnet_w48 | Composition-1k |  通用目标抠图 | 305M | [推理模型](https://paddleseg.bj.bcebos.com/matting/models/deploy/ppmatting-hrnet_w48-composition.zip)/[预训练模型](https://paddleseg.bj.bcebos.com/matting/models/ppmatting-hrnet_w48-composition.pdparams) |
+|ppmatting_hrnet_w48 | Distinctions-646 | 通用目标抠图 | 305M | [推理模型](https://paddleseg.bj.bcebos.com/matting/models/deploy/ppmatting-hrnet_w48-distinctions.zip)/[预训练模型](https://paddleseg.bj.bcebos.com/matting/models/ppmatting-hrnet_w48-distinctions.pdparams) |
+
+## 人像抠图
+|模型名称  | 模型简介 |  推理模型大小 | 下载地址 |
+|---|---|---|---|
+|ppmatting_hrnet_w18 | 人像抠图 | 87M | [推理模型](https://paddleseg.bj.bcebos.com/matting/models/deploy/ppmatting-hrnet_w18-human_512.zip)/[预训练模型](https://paddleseg.bj.bcebos.com/matting/models/ppmatting-hrnet_w18-human_512.pdparams) |

modelcenter/PP-Matting/info.yaml

+---
+Model_Info:
+  name: "PP-Matting"
+  description: "PP-Matting图像抠图模型"
+  description_en: "PP-Matting image matting model"
+  icon: "@后续UE统一设计之后，会存到bos上某个位置"
+  from_repo: "PaddleSeg"
+Task:
+- tag_en: "Computer Vision"
+  tag: "计算机视觉"
+  sub_tag_en: "Image Matting, Figure Cutout"
+  sub_tag: "图像抠图, 人像抠图"
+Example:
+- title: "【PaddleSeg-Matting实践范例】PP-Matting图像抠图"
+  url: "https://aistudio.baidu.com/aistudio/projectdetail/5002963"
+Datasets: "Composition-1k, Distinctions-646"
+Pulisher: "Baidu"
+License: "apache.2.0"
+Paper:
+- title: "PP-Matting: High-Accuracy Natural Image Matting"
+  url: "https://arxiv.org/abs/2204.09433"
+IfTraining: 1
+IfOnlineDemo: 1

modelcenter/PP-Matting/introduction_cn.ipynb

+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "03cfffa3-2398-4d55-bf1e-fe3e01f10d68",
+   "metadata": {},
+   "source": [
+    "## 1. PP-Matting 模型简介\n",
+    "\n",
+    "在众多图像抠图算法中，为了追求精度，往往需要输入trimap作为辅助信息，但这极大限制了算法的使用性。PP-Matting作为一种trimap-free的抠图方法，有效克服了辅助信息带来的弊端，在Composition-1k和Distinctions-646数据集中取得了SOTA的效果。PP-Matting利用语义分支（SCB）提取图片高级语义信息并通过引导流设计（Guidance Flow）逐步引导高分辨率细节分支（HRBP）对过度区域的细节提取，最后通过融合模块实现语义和细节的融合得到最终的alpha matte。\n",
+    "\n",
+    "更多细节可参考技术报告：https://arxiv.org/abs/2204.09433 。\n",
+    "\n",
+    "更多关于PaddleMatting的内容，可以点击 https://github.com/PaddlePaddle/PaddleSeg/tree/develop/Matting 进行了解。\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8cbcf510-dc56-43f9-9864-5e1de3c7b272",
+   "metadata": {},
+   "source": [
+    "## 2. 模型效果及应用场景\n",
+    "PP-Matting在人像上的抠图效果如下：\n",
+    "<p align=\"center\">\n",
+    "<img src=\"https://user-images.githubusercontent.com/30919197/179751613-d26f2261-7bcf-4066-a0a4-4c818e7065f0.gif\" width=\"100%\" height=\"100%\">\n",
+    "</p>\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8d9da224-edd4-4c9e-ab2d-cba7ee5a92a4",
+   "metadata": {},
+   "source": [
+    "## 3. 模型如何使用"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "2ac57be7-c00f-441e-ad82-635f6268b6bd",
+   "metadata": {},
+   "source": [
+    "### 3.1 模型推理\n",
+    "* 下载"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3ea22cb4-5bed-4ce0-858b-3bbb342e8ccf",
+   "metadata": {
+    "scrolled": true,
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "%cd ~/work\n",
+    "!git clone --depth 1  https://gitee.com/paddlepaddle/PaddleSeg"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "fd7142e9-5202-4a4d-8a03-36fcc5ea0259",
+   "metadata": {},
+   "source": [
+    "* 安装"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a7fd9575-4116-4fa7-86ee-0b2db7417300",
+   "metadata": {
+    "scrolled": true,
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "%cd ~/work/PaddleSeg/Matting\n",
+    "!pip install -r requirements.txt"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a0dd2390-e635-432b-9e0b-d7c9323f81a9",
+   "metadata": {},
+   "source": [
+    "* 快速体验\n",
+    "恭喜！ 您已经成功安装了PaddleSeg，接下来快速体验图像抠图效果"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "81736dd2-e90d-4dac-b3c0-58424e2e8dc4",
+   "metadata": {
+    "scrolled": true,
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# 下载预训练模型\n",
+    "!wget https://paddleseg.bj.bcebos.com/matting/models/ppmatting-hrnet_w18-human_512.pdparams\n",
+    "# 下载图片\n",
+    "!wget https://user-images.githubusercontent.com/30919197/200645066-6898ec5a-f1c5-4bf7-aa41-473a29977a86.jpeg\n",
+    "# 在GPU上预测一张图片\n",
+    "!export CUDA_VISIBLE_DEVICES=0\n",
+    "!python tools/predict.py \\\n",
+    "    --config configs/ppmatting/ppmatting-hrnet_w18-human_512.yml \\\n",
+    "    --model_path ppmatting-hrnet_w18-human_512.pdparams \\\n",
+    "    --image_path 200645066-6898ec5a-f1c5-4bf7-aa41-473a29977a86.jpeg \\\n",
+    "    --save_dir ./output/results \\\n",
+    "    --fg_estimate True"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3ab60ad4-2908-40e2-9c08-34ac7978ef16",
+   "metadata": {},
+   "source": [
+    "会在output/results文件夹下生成一个alpha图像和一个rgba图像。\n",
+    "\n",
+    "结果如下图：\n",
+    "<div align=\"center\">\n",
+    "<img src=\"https://user-images.githubusercontent.com/30919197/200647513-e744118a-bd2b-4de2-b740-6eaeaccf47b6.png\"  width = \"40%\"  /> \n",
+    "<img src=\"https://user-images.githubusercontent.com/30919197/200647492-7fd5fd80-2c2a-4775-ad9e-fd1d476cba48.png\"  width = \"40%\"  />\n",
+    "* </div>\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "98c083d9-acd1-4e0a-855b-0a048983f251",
+   "metadata": {},
+   "source": [
+    "### 3.2 模型训练\n",
+    " * 克隆PaddleSeg仓库（详见3.1）。\n",
+    " \n",
+    " 具体训练参考PaddleSeg仓库Matting子目录相关教程\n",
+    "https://github.com/PaddlePaddle/PaddleSeg/tree/develop/Matting 。"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "e9b05c1e-cb59-4dd7-83c0-cb30dfefba60",
+   "metadata": {},
+   "source": [
+    "## 4. 原理\n",
+    "\n",
+    "<div align=\"center\">\n",
+    "<img src=\"https://user-images.githubusercontent.com/30919197/200649929-3fa07082-7980-49f8-9f7f-38c31c46de20.png\"  width = \"80%\"  />\n",
+    "* </div>\n",
+    "\n",
+    "* 分支设计，明确语义预测和细节预测任务。\n",
+    "* SCB(Semantic Context Branch)分支进行语义预测，保证图像整体预测正确性，其将图像粗略的分为三个部分，即前景、背景和过度区域。\n",
+    "* HRDB（High-Resolution Detail Branch）维持高分辨率特征提取过程，保证细节不受损失。\n",
+    "* 引导流结构设计（Guidance Flow）使HRBD分支获取SCB分支提取的语义信息，使HRDB分支能专注与过度区域的细节预测。\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ae7e1030-9044-44f5-a0d3-5148e59d4753",
+   "metadata": {},
+   "source": [
+    "## 5. 注意事项\n",
+    "* 基于公开数据集Composition-1K和Distinctions-646的训练和预测需发邮件向作者申请。\n",
+    "* PP-Matting开源了人像预训练模型，可根据具体的人像抠图场景，标注少量的数据进行finetune。"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7e238357-2f19-48c3-902e-9ba3c93f7818",
+   "metadata": {},
+   "source": [
+    "## 6. 相关论文以及引用信息\n",
+    "@article{chen2022pp,\n",
+    "  title={PP-Matting: High-Accuracy Natural Image Matting},\n",
+    "  author={Chen, Guowei and Liu, Yi and Wang, Jian and Peng, Juncai and Hao, Yuying and Chu, Lutao and Tang, Shiyu and Wu, Zewu and Chen, Zeyu and Yu, Zhiliang and others},\n",
+    "  journal={arXiv preprint arXiv:2204.09433},\n",
+    "  year={2022}\n",
+    "}"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "py35-paddle1.2.0"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}