From 3ce9dd26c6f0b1b77263669ff3552b77c230ef15 Mon Sep 17 00:00:00 2001
From: xiaoting <31891223+tink2123@users.noreply.github.com>
Date: Tue, 24 May 2022 21:57:14 +0800
Subject: [PATCH] update serving tipc (#5529)

* update serving tipc

* rm python version

* rm python version

* Update serving_python.md

* mv unset to prepare.sh
---
 .../Step6/deploy/serving_python/config.yml    |  2 +-
 .../deploy/serving_python/web_service.py      | 63 +++++++++++++-
 .../serving_infer_python.txt                  |  3 +-
 .../Step6/test_tipc/prepare.sh                |  2 +
 .../test_tipc/test_serving_infer_python.sh    | 56 ++++++++-----
 tutorials/tipc/serving_cpp/serving_cpp.md     | 82 +++++++++++++------
 .../template/code/serving_client.py           | 13 +--
 .../tipc/serving_python/serving_python.md     | 14 ++--
 8 files changed, 170 insertions(+), 65 deletions(-)

diff --git a/tutorials/mobilenetv3_prod/Step6/deploy/serving_python/config.yml b/tutorials/mobilenetv3_prod/Step6/deploy/serving_python/config.yml
index 88957745..469dca5b 100755
--- a/tutorials/mobilenetv3_prod/Step6/deploy/serving_python/config.yml
+++ b/tutorials/mobilenetv3_prod/Step6/deploy/serving_python/config.yml
@@ -22,7 +22,7 @@ op:
             model_config: "./serving_server"
 
             #计算硬件类型: 空缺时由devices决定(CPU/GPU)，0=cpu, 1=gpu, 2=tensorRT, 3=arm cpu, 4=kunlun xpu
-            device_type: 1
+            device_type: 
 
             #计算硬件ID，当devices为""或不写时为CPU预测；当devices为"0", "0,1,2"时为GPU预测，表示使用的GPU卡
             devices: "0" # "0,1"
diff --git a/tutorials/mobilenetv3_prod/Step6/deploy/serving_python/web_service.py b/tutorials/mobilenetv3_prod/Step6/deploy/serving_python/web_service.py
index 94beb893..4d8102d6 100755
--- a/tutorials/mobilenetv3_prod/Step6/deploy/serving_python/web_service.py
+++ b/tutorials/mobilenetv3_prod/Step6/deploy/serving_python/web_service.py
@@ -21,6 +21,62 @@ from PIL import Image
 import io
 from preprocess_ops import ResizeImage, CenterCropImage, NormalizeImage, ToCHW, Compose
 
+from argparse import ArgumentParser, RawDescriptionHelpFormatter
+import yaml
+
+
+class ArgsParser(ArgumentParser):
+    def __init__(self):
+        super(ArgsParser, self).__init__(
+            formatter_class=RawDescriptionHelpFormatter)
+        self.add_argument("-c", "--config", help="configuration file to use")
+        self.add_argument(
+            "-o", "--opt", nargs='+', help="set configuration options")
+
+    def parse_args(self, argv=None):
+        args = super(ArgsParser, self).parse_args(argv)
+        assert args.config is not None, \
+            "Please specify --config=configure_file_path."
+        args.conf_dict = self._parse_opt(args.opt, args.config)
+        print("args config:", args.conf_dict)
+        return args
+
+    def _parse_helper(self, v):
+        if v.isnumeric():
+            if "." in v:
+                v = float(v)
+            else:
+                v = int(v)
+        elif v == "True" or v == "False":
+            v = (v == "True")
+        return v
+
+    def _parse_opt(self, opts, conf_path):
+        f = open(conf_path)
+        config = yaml.load(f, Loader=yaml.Loader)
+        if not opts:
+            return config
+        for s in opts:
+            s = s.strip()
+            k, v = s.split('=')
+            v = self._parse_helper(v)
+            if "devices" in k:
+                v = str(v)
+            print(k, v, type(v))
+            cur = config
+            parent = cur
+            for kk in k.split("."):
+                if kk not in cur:
+                    cur[kk] = {}
+                    parent = cur
+                    cur = cur[kk]
+                else:
+                    parent = cur
+                    cur = cur[kk]
+            parent[k.split(".")[-1]] = v
+        return config
+
+
 class MobileNetV3Op(Op):
     def init_op(self):
         self.seq = Compose([
@@ -53,16 +109,17 @@ class MobileNetV3Op(Op):
         result["prob"] = str(result["prob"])
         return result, None, ""
 
+
 class MobileNetV3Service(WebService):
     def get_pipeline_response(self, read_op):
-        mobilenetv3_op = MobileNetV3Op(
-            name="imagenet", input_ops=[read_op])
+        mobilenetv3_op = MobileNetV3Op(name="imagenet", input_ops=[read_op])
         return mobilenetv3_op
 
 
 # define the service class
 uci_service = MobileNetV3Service(name="imagenet")
 # load config and prepare the service
-uci_service.prepare_pipeline_config("config.yml")
+FLAGS = ArgsParser().parse_args()
+uci_service.prepare_pipeline_config(yml_dict=FLAGS.conf_dict)
 # start the service
 uci_service.run_service()
diff --git a/tutorials/mobilenetv3_prod/Step6/test_tipc/configs/mobilenet_v3_small/serving_infer_python.txt b/tutorials/mobilenetv3_prod/Step6/test_tipc/configs/mobilenet_v3_small/serving_infer_python.txt
index e59a41e3..c59180eb 100644
--- a/tutorials/mobilenetv3_prod/Step6/test_tipc/configs/mobilenet_v3_small/serving_infer_python.txt
+++ b/tutorials/mobilenetv3_prod/Step6/test_tipc/configs/mobilenet_v3_small/serving_infer_python.txt
@@ -8,6 +8,7 @@ trans_model:-m paddle_serving_client.convert
 --serving_server:./deploy/serving_python/serving_server/
 --serving_client:./deploy/serving_python/serving_client/
 serving_dir:./deploy/serving_python
-web_service:web_service.py
+web_service:web_service.py --config=config.yml 
+--opt op.imagenet.local_service_conf.devices:"0"|null
 pipline:pipeline_http_client.py
 --image_dir:../../lite_data/test/
\ No newline at end of file
diff --git a/tutorials/mobilenetv3_prod/Step6/test_tipc/prepare.sh b/tutorials/mobilenetv3_prod/Step6/test_tipc/prepare.sh
index bc175830..e013ec5d 100644
--- a/tutorials/mobilenetv3_prod/Step6/test_tipc/prepare.sh
+++ b/tutorials/mobilenetv3_prod/Step6/test_tipc/prepare.sh
@@ -54,6 +54,8 @@ elif [ ${MODE} = "serving_infer" ];then
         wget -nc -P  ./inference https://paddle-model-ecology.bj.bcebos.com/model/mobilenetv3_reprod/mobilenet_v3_small_infer.tar  --no-check-certificate
         cd ./inference && tar xf mobilenet_v3_small_infer.tar && cd ../
     fi
+    unset https_proxy
+    unset http_proxy
 elif [ ${MODE} = "cpp_infer" ];then
     PADDLEInfer=$3
     # wget model
diff --git a/tutorials/mobilenetv3_prod/Step6/test_tipc/test_serving_infer_python.sh b/tutorials/mobilenetv3_prod/Step6/test_tipc/test_serving_infer_python.sh
index 8e79e257..344352f1 100644
--- a/tutorials/mobilenetv3_prod/Step6/test_tipc/test_serving_infer_python.sh
+++ b/tutorials/mobilenetv3_prod/Step6/test_tipc/test_serving_infer_python.sh
@@ -25,9 +25,11 @@ serving_client_key=$(func_parser_key "${lines[8]}")
 serving_client_value=$(func_parser_value "${lines[8]}")
 serving_dir_value=$(func_parser_value "${lines[9]}")
 web_service_py=$(func_parser_value "${lines[10]}")
-pipeline_py=$(func_parser_value "${lines[11]}")
-image_dir_key=$(func_parser_key "${lines[12]}")
-image_dir_value=$(func_parser_value "${lines[12]}")
+web_use_gpu_key=$(func_parser_key "${lines[11]}")
+web_use_gpu_list=$(func_parser_value "${lines[11]}")
+pipeline_py=$(func_parser_value "${lines[12]}")
+image_dir_key=$(func_parser_key "${lines[13]}")
+image_dir_value=$(func_parser_value "${lines[13]}")
 
 
 LOG_PATH="./log/${model_name}/${MODE}"
@@ -53,24 +55,38 @@ function func_serving(){
     cd ${serving_dir_value}
     status_check $last_status "${trans_model_cmd}" "${status_log}" "${model_name}"
     echo $PWD
-    unset https_proxy
-    unset http_proxy
 
-    _save_log_path="../../log/${model_name}/${MODE}/serving_infer_python_gpu_batchsize_1.log"
-    web_service_cmd="${python} ${web_service_py} &"
-    eval $web_service_cmd
-    last_status=${PIPESTATUS[0]}
-    status_check $last_status "${web_service_cmd}" "${status_log}" "${model_name}"
-    sleep 5s
-    _save_log_path="../../log/${model_name}/${MODE}/server_infer_gpu_batchsize_1.log"
-    set_image_dir=$(func_set_params "${image_dir_key}" "${image_dir_value}")
-    pipeline_cmd="${python} ${pipeline_py} ${set_image_dir} > ${_save_log_path} 2>&1 "
-    eval $pipeline_cmd
-    last_status=${PIPESTATUS[0]}
-    status_check $last_status "${pipeline_cmd}" "${status_log}" "${model_name}"
-    eval "cat ${_save_log_path}"
-    cd ../../
-    ps ux | grep -E 'web_service|pipeline' | awk '{print $2}' | xargs kill -s 9
+    for use_gpu in ${web_use_gpu_list[*]}; do
+        if [ ${use_gpu} = "null" ]; then
+            _save_log_path="../../log/${model_name}/${MODE}/serving_infer_python_cpu_batchsize_1.log"
+            web_service_cmd="${python} ${web_service_py} ${web_use_gpu_key}="" &"
+            eval $web_service_cmd
+            last_status=${PIPESTATUS[0]}
+            status_check $last_status "${web_service_cmd}" "${status_log}" "${model_name}"
+            sleep 5s
+            set_image_dir=$(func_set_params "${image_dir_key}" "${image_dir_value}")
+            pipeline_cmd="${python} ${pipeline_py} ${set_image_dir} > ${_save_log_path} 2>&1 "
+            eval $pipeline_cmd
+            last_status=${PIPESTATUS[0]}
+            status_check $last_status "${pipeline_cmd}" "${status_log}" "${model_name}"
+            eval "cat ${_save_log_path}"
+            ps ux | grep -E 'web_service|pipeline' | awk '{print $2}' | xargs kill -s 9
+        else
+            _save_log_path="../../log/${model_name}/${MODE}/serving_infer_python_gpu_batchsize_1.log"
+            web_service_cmd="${python} ${web_service_py} ${web_use_gpu_key}=${use_gpu} &"
+            eval $web_service_cmd
+            last_status=${PIPESTATUS[0]}
+            status_check $last_status "${web_service_cmd}" "${status_log}" "${model_name}"
+            sleep 5s
+            set_image_dir=$(func_set_params "${image_dir_key}" "${image_dir_value}")
+            pipeline_cmd="${python} ${pipeline_py} ${set_image_dir} > ${_save_log_path} 2>&1 "
+            eval $pipeline_cmd
+            last_status=${PIPESTATUS[0]}
+            status_check $last_status "${pipeline_cmd}" "${status_log}" "${model_name}"
+            eval "cat ${_save_log_path}"
+            ps ux | grep -E 'web_service|pipeline' | awk '{print $2}' | xargs kill -s 9
+        fi
+    done
 }
 
 
diff --git a/tutorials/tipc/serving_cpp/serving_cpp.md b/tutorials/tipc/serving_cpp/serving_cpp.md
index 3d4901c1..f560a28c 100644
--- a/tutorials/tipc/serving_cpp/serving_cpp.md
+++ b/tutorials/tipc/serving_cpp/serving_cpp.md
@@ -71,15 +71,13 @@ Paddle Serving服务化部署主要包括以下步骤：
   nvidia-docker exec -it test bash
   ```
 
-（2）安装Paddle Serving四个安装包，分别是：paddle-serving-server(CPU/GPU版本二选一), paddle-serving-client, paddle-serving-app和paddlepaddle(CPU/GPU版本二选一)。
+（2）安装Paddle Serving三个安装包，分别是：paddle-serving-server(CPU/GPU版本二选一), paddle-serving-client, paddle-serving-app和paddlepaddle(CPU/GPU版本二选一)。
 
   ```bash
-  pip3 install paddle-serving-client==0.8.3
-  #pip3 install paddle-serving-server==0.8.3 # CPU
-  pip3 install paddle-serving-server-gpu==0.8.3.post102 # GPU with CUDA10.2 + TensorRT6
-  pip3 install paddle-serving-app==0.8.3
-  #pip3 install paddlepaddle==2.2.2 # CPU
-  pip3 install paddlepaddle-gpu==2.2.2
+  pip install paddle-serving-client
+  pip install paddle-serving-app
+  #pip install paddlepaddle # CPU
+  pip install paddlepaddle-gpu
   ```
   您可能需要使用国内镜像源（例如百度源, 在pip命令中添加`-i https://mirror.baidu.com/pypi/simple`）来加速下载。
   Paddle Serving Server更多不同运行环境的whl包下载地址，请参考：[下载页面](https://github.com/PaddlePaddle/Serving/blob/v0.8.3/doc/Latest_Packages_CN.md)
@@ -105,7 +103,7 @@ cd models/tutorials/tipc/serving_cpp
 为了便于模型服务化部署，需要将静态图模型(模型结构文件：\*.pdmodel和模型参数文件：\*.pdiparams)使用paddle_serving_client.convert按如下命令转换为服务化部署模型：
 
 ```bash
-python3 -m paddle_serving_client.convert --dirname {静态图模型路径} --model_filename {模型结构文件} --params_filename {模型参数文件} --serving_server {转换后的服务器端模型和配置文件存储路径} --serving_client {转换后的客户端模型和配置文件存储路径}
+python -m paddle_serving_client.convert --dirname {静态图模型路径} --model_filename {模型结构文件} --params_filename {模型参数文件} --serving_server {转换后的服务器端模型和配置文件存储路径} --serving_client {转换后的客户端模型和配置文件存储路径}
 ```
 上面命令中 "转换后的服务器端模型和配置文件" 将用于后续服务化部署。其中`paddle_serving_client.convert`命令是`paddle_serving_client` whl包内置的转换函数，无需修改。
 
@@ -114,7 +112,7 @@ python3 -m paddle_serving_client.convert --dirname {静态图模型路径} --mod
 针对MobileNetV3网络，将inference模型转换为服务化部署模型的示例命令如下，转换完后在本地生成**serving_server**和**serving_client**两个文件夹。本教程后续主要使用serving_server文件夹中的模型。
 
 ```bash
-python3 -m paddle_serving_client.convert \
+python -m paddle_serving_client.convert \
     --dirname ./mobilenet_v3_small_infer/ \
     --model_filename inference.pdmodel \
     --params_filename inference.pdiparams \
@@ -133,7 +131,6 @@ python3 -m paddle_serving_client.convert \
 
 - serving_client.py：用于**客户端**访问服务的程序，开发者需要设置url（服务地址）、logid（日志ID）和测试图像。
 
-- preprocess_ops.py：用户图片前项处理的一些工具类
 
 **【实战】**
 
@@ -145,7 +142,40 @@ cp -r ./template/code/*  ./
 <a name="2.5"></a>
 ### 2.5 服务端修改
 
-服务端无需进行代码修改。
+需要将前后处理部分以自定义OP的形式与 Serving 服务端编译在一起，推荐复用 cpp_infer 中的代码。
+
+**【基本流程】**
+
+具体开发流程可参考[新增op](https://github.com/PaddlePaddle/Serving/blob/v0.8.3/doc/C%2B%2B_Serving/2%2B_model.md)
+
+**【实战】**
+
+针对MobileNet网络, 新增三类文件:
+
+* [General_Clas_op.*](../../mobilenetv3_prod/Step6/deploy/serving_cpp/preprocess/general_clas_op.cpp) 包含自定义op部分，读取 client 端传输数据并进行推理预测；
+
+* [preprocess_op.*](../../mobilenetv3_prod/Step6/deploy/serving_cpp/preprocess/preprocess_op.cpp) 是可能会用到的工具类函数，复用 cpp_infer 中的代码；
+
+* [serving_client_conf.prototxt](../../mobilenetv3_prod/Step6/deploy/serving_cpp/preprocess/serving_client_conf.prototxt) 由于clint端输入的是原始图像，可能与推理时需要的输入数据类型不同，建议将输入数据类型统一修改成string，tipc测试时每次copy该文件，覆盖自动生成的`serving_client_conf.prototxt`。 具体将 feed_var 中的 feed_type 修改为20，shape修改为1。
+
+```
+feed_var {
+  name: "input"
+  alias_name: "input"
+  is_lod_tensor: false
+  feed_type: 20
+  shape: 1
+}
+```
+
+完成代码开发后，将相关代码cp到Serving源码中并进行编译[编译](https://github.com/PaddlePaddle/Serving/blob/v0.8.3/doc/Compile_CN.md)：
+
+```
+cp deploy/serving_cpp/preprocess/general_clas_op.* ${Serving_repo_path}/core/general-server/op
+cp deploy/serving_cpp/preprocess/preprocess_op.* ${Serving_repo_path}/core/predictor/tools/pp_shitu_tools
+```
+
+
 
 <a name="2.6"></a>
 ### 2.6 客户端修改
@@ -170,13 +200,11 @@ img_path = "./images/demo.jpg"
 
 **【基本流程】**
 
-serving_client.py文件中的preprocess函数用于开发数据预处理程序，包含输入、处理流程和输出三部分。
+serving_client.py文件中的preprocess函数用于开发数据处理程序，包含输入和输出两部分。
 
 **（1）输入：** 一般开发者使用时，需要自行定义读取数据的方式。
 
-**（2）处理流程：** 数据预处理流程和基于Paddle Inference的模型预处理流程相同。
-
-**（3）输出：** 需要返回两个参数，第一个参数为模型预测的输入字典，第二个参数为需要获取的模型输出列表
+**（2）输出：** 需要返回两个参数，第一个参数为模型预测的输入字典，第二个参数为需要获取的模型输出列表
 
 ```
 {"input": input_data}, ["output"]
@@ -196,20 +224,20 @@ import numpy as np
 import base64
 from PIL import Image
 import io
-from preprocess_ops import ResizeImage, CenterCropImage, NormalizeImage, ToCHW, Compose
 from paddle_serving_client import Client
 ```  
-添加数据读取相关代码：
+添加数据读取相关代码，将输入图片转换成string类型：
 
 ```py
-def preprocess():
-    image_file = img_path
-    image = Image.open(image_file)
-    seq = Compose([
-            ResizeImage(256), CenterCropImage(224), NormalizeImage(), ToCHW()
-        ])
-    input_data=seq(image)
-    feed = {"input": input_data}
+def cv2_to_base64(image):
+    return base64.b64encode(image).decode(
+        'utf8')
+
+def preprocess(img_file):
+    with open(img_file, 'rb') as file:
+        image_data = file.read()
+    image = cv2_to_base64(image_data)
+    feed = {"input": image}
     fetch = ["softmax_1.tmp_0"]
     return feed, fetch
 ```
@@ -253,7 +281,7 @@ def postprocess(fetch_map):
 可以按如下命令启动模型预测服务：
 
 ```bash
-python3 -m paddle_serving_server.serve --model serving_server --port 9993
+python -m paddle_serving_server.serve --model serving_server --op GeneralClasOp --port 9993
 ```  
 **【实战】**
 
@@ -273,7 +301,7 @@ python3 -m paddle_serving_server.serve --model serving_server --port 9993
 客户端访问服务的命令如下：
 
 ```bash
-python3 serving_client.py
+python serving_client.py
 ```  
 访问成功的界面如下图：
 
diff --git a/tutorials/tipc/serving_cpp/template/code/serving_client.py b/tutorials/tipc/serving_cpp/template/code/serving_client.py
index 765efa20..a9fa861a 100755
--- a/tutorials/tipc/serving_cpp/template/code/serving_client.py
+++ b/tutorials/tipc/serving_cpp/template/code/serving_client.py
@@ -13,11 +13,13 @@
 # limitations under the License.
 
 from paddle_serving_client import Client
+import os
 
 url = "127.0.0.1:9993"
 logid = 10000
 img_path = "./images/demo.jpg"
 
+
 def preprocess():
     """preprocess
         
@@ -30,6 +32,7 @@ def preprocess():
     """
     pass
 
+
 def postprocess(fetch_map):
     """postprocess
 
@@ -42,14 +45,12 @@ def postprocess(fetch_map):
     """
     pass
 
+
 client = Client()
-client.load_client_config(
-    "serving_client/serving_client_conf.prototxt")
+client.load_client_config("serving_client/serving_client_conf.prototxt")
 client.connect([url])
 
-feed, fetch = preprocess()
-
+feed, fetch = preprocess(img_path)
 fetch_map = client.predict(feed=feed, fetch=fetch)
-
 result = postprocess(fetch_map)
-print(result)
\ No newline at end of file
+print(result)
diff --git a/tutorials/tipc/serving_python/serving_python.md b/tutorials/tipc/serving_python/serving_python.md
index 1249304a..b67a7391 100644
--- a/tutorials/tipc/serving_python/serving_python.md
+++ b/tutorials/tipc/serving_python/serving_python.md
@@ -76,15 +76,15 @@ Paddle Serving服务化部署主要包括以下步骤：
   nvidia-docker exec -it test bash
   ```
 
-（2）安装Paddle Serving四个安装包，分别是：paddle-serving-server(CPU/GPU版本二选一), paddle-serving-client, paddle-serving-app和paddlepaddle(CPU/GPU版本二选一)。
+（2）安装Paddle Serving四个安装包的最新版本，分别是：paddle-serving-server(CPU/GPU版本二选一), paddle-serving-client, paddle-serving-app和paddlepaddle(CPU/GPU版本二选一)。
 
   ```bash
-  pip3 install paddle-serving-client==0.7.0
-  #pip3 install paddle-serving-server==0.7.0 # CPU
-  pip3 install paddle-serving-server-gpu==0.7.0.post102 # GPU with CUDA10.2 + TensorRT6
-  pip3 install paddle-serving-app==0.7.0
-  #pip3 install paddlepaddle==2.2.1 # CPU
-  pip3 install paddlepaddle-gpu==2.2.1
+  pip install paddle-serving-client
+  #pip install paddle-serving-server # CPU
+  pip install paddle-serving-server-gpu # GPU 默认 CUDA10.2 + TensorRT6，其他环境需手动指定版本号
+  pip install paddle-serving-app
+  #pip install paddlepaddle # CPU
+  pip install paddlepaddle-gpu
   ```
   您可能需要使用国内镜像源（例如百度源, 在pip命令中添加`-i https://mirror.baidu.com/pypi/simple`）来加速下载。
   Paddle Serving Server更多不同运行环境的whl包下载地址，请参考：[下载页面](https://github.com/PaddlePaddle/Serving/blob/v0.7.0/doc/Latest_Packages_CN.md)
-- 
GitLab