update serving

deploy/pdserving/README_CN.md

@@ -29,7 +29,9 @@ PaddleOCR提供2种服务部署方式：
 
 需要准备PaddleOCR的运行环境和Paddle Serving的运行环境。
 
-- 准备PaddleOCR的运行环境参考[链接](../../doc/doc_ch/installation.md)
+- 准备PaddleOCR的运行环境[链接](../../doc/doc_ch/installation.md)
+  根据环境下载对应的paddle whl包，
+  推荐2.0.1版本：https://www.paddlepaddle.org.cn/whl/mkl/stable.html
 
 - 准备PaddleServing的运行环境，步骤如下
 
@@ -45,25 +47,16 @@ PaddleOCR提供2种服务部署方式：
     ```
 
 2. 安装client，用于向服务发送请求
-    ```
-    pip3 install paddle-serving-client==0.5.0  # for CPU
+    在[下载链接](https://github.com/PaddlePaddle/Serving/blob/develop/doc/LATEST_PACKAGES.md)中找到对应python版本的client安装包，这里推荐python3.7版本：
 
-    pip3 install paddle-serving-client-gpu==0.5.0   # for GPU
+    ```
+    wget https://paddle-serving.bj.bcebos.com/whl/paddle_serving_client-0.0.0-cp37-none-any.whl
+    pip3 install paddle_serving_client-0.0.0-cp37-none-any.whl
     ```
 
 3. 安装serving-app
     ```
-    pip3 install paddle-serving-app==0.3.0
-    ```
-    **note:**  安装0.3.0版本的serving-app后，为了能加载动态图模型，需要修改serving_app的源码，具体为：
-    ```
-    # 找到paddle_serving_app的安装目录，找到并编辑local_predict.py文件
-    vim /usr/local/lib/python3.7/site-packages/paddle_serving_app/local_predict.py
-    # 将local_predict.py 的第85行 config = AnalysisConfig(model_path)  替换为：
-    if os.path.exists(os.path.join(model_path, "__params__")):
-        config = AnalysisConfig(os.path.join(model_path, "__model__"), os.path.join(model_path, "__params__"))
-    else:
-        config = AnalysisConfig(model_path)
+    pip3 install paddle-serving-app==0.3.1
     ```
 
     **Note:** 如果要安装最新版本的PaddleServing参考[链接](https://github.com/PaddlePaddle/Serving/blob/develop/doc/LATEST_PACKAGES.md)。

deploy/pdserving/config.yml

 #rpc端口, rpc_port和http_port不允许同时为空。当rpc_port为空且http_port不为空时，会自动将rpc_port设置为http_port+1
-rpc_port: 18090
+rpc_port: 18091
 
 #http端口, rpc_port和http_port不允许同时为空。当rpc_port可用且http_port为空时，不自动生成http_port
-http_port: 9999
+http_port: 9998
 
 #worker_num, 最大并发数。当build_dag_each_worker=True时, 框架会创建worker_num个进程，每个进程内构建grpcSever和DAG
 ##当build_dag_each_worker=False时，框架会设置主线程grpc线程池的max_workers=worker_num
-worker_num: 20
+worker_num: 5
 
 #build_dag_each_worker, False，框架在进程内创建一条DAG；True，框架会每个进程内创建多个独立的DAG
-build_dag_each_worker: false
+build_dag_each_worker: False
 
 dag:
     #op资源类型, True, 为线程模型；False，为进程模型
     is_thread_op: False
 
     #重试次数
-    retry: 1
+    retry: 10
 
     #使用性能分析, True，生成Timeline性能数据，对性能有一定影响；False为不使用
-    use_profile: False
+    use_profile: True
     
     tracer:
         interval_s: 10
 op:
     det:
         #并发数，is_thread_op=True时，为线程并发；否则为进程并发
-        concurrency: 4
+        concurrency: 2
 
         #当op配置没有server_endpoints时，从local_service_conf读取本地服务配置
         local_service_conf:
@@ -34,15 +34,15 @@ op:
             client_type: local_predictor
 
             #det模型路径
-            model_config: /paddle/serving/models/det_serving_server/ #ocr_det_model
+            model_config: ./ppocr_det_server_2.0_serving
 
             #Fetch结果列表，以client_config中fetch_var的alias_name为准
             fetch_list: ["save_infer_model/scale_0.tmp_1"]
 
             #计算硬件ID，当devices为""或不写时为CPU预测；当devices为"0", "0,1,2"时为GPU预测，表示使用的GPU卡
-            devices: "2"
+            devices: "0"
             
-            ir_optim: True
+            ir_optim: False
     rec:
         #并发数，is_thread_op=True时，为线程并发；否则为进程并发
         concurrency: 1
@@ -60,12 +60,12 @@ op:
             client_type: local_predictor
 
             #rec模型路径
-            model_config: /paddle/serving/models/rec_serving_server/ #ocr_rec_model
+            model_config: ./ppocr_rec_server_2.0_serving
 
             #Fetch结果列表，以client_config中fetch_var的alias_name为准
-            fetch_list: ["save_infer_model/scale_0.tmp_1"] #["ctc_greedy_decoder_0.tmp_0", "softmax_0.tmp_0"] 
+            fetch_list: ["save_infer_model/scale_0.tmp_1"]  
 
             #计算硬件ID，当devices为""或不写时为CPU预测；当devices为"0", "0,1,2"时为GPU预测，表示使用的GPU卡
-            devices: "2"
+            devices: "0"
             
-            ir_optim: True
+            ir_optim: False

deploy/pdserving/ocr_reader.py

@@ -21,7 +21,6 @@ import sys
 import argparse
 import string
 from copy import deepcopy
-import paddle
 
 
 class DetResizeForTest(object):
@@ -227,8 +226,8 @@ class CTCLabelDecode(BaseRecLabelDecode):
         super(CTCLabelDecode, self).__init__(config)
 
     def __call__(self, preds, label=None, *args, **kwargs):
-        if isinstance(preds, paddle.Tensor):
-            preds = preds.numpy()
+        #if isinstance(preds, paddle.Tensor):
+        #    preds = preds.numpy()
         preds_idx = preds.argmax(axis=2)
         preds_prob = preds.max(axis=2)
         text = self.decode(preds_idx, preds_prob, is_remove_duplicate=True)

deploy/pdserving/pipeline_http_client.py

@@ -23,8 +23,8 @@ def cv2_to_base64(image):
     return base64.b64encode(image).decode('utf8')
 
 
-url = "http://127.0.0.1:9999/ocr/prediction"
-test_img_dir = "../doc/imgs/"
+url = "http://127.0.0.1:9998/ocr/prediction"
+test_img_dir = "../../doc/imgs/"
 for idx, img_file in enumerate(os.listdir(test_img_dir)):
     with open(os.path.join(test_img_dir, img_file), 'rb') as file:
         image_data1 = file.read()
@@ -36,5 +36,5 @@ for idx, img_file in enumerate(os.listdir(test_img_dir)):
         r = requests.post(url=url, data=json.dumps(data))
         print(r.json())
 
-test_img_dir = "../doc/imgs/"
+test_img_dir = "../../doc/imgs/"
 print("==> total number of test imgs: ", len(os.listdir(test_img_dir)))

deploy/pdserving/pipeline_rpc_client.py

@@ -23,7 +23,7 @@ import base64
 import os
 
 client = PipelineClient()
-client.connect(['127.0.0.1:18090'])
+client.connect(['127.0.0.1:18091'])
 
 
 def cv2_to_base64(image):
@@ -39,4 +39,3 @@ for img_file in os.listdir(test_img_dir):
 for i in range(1):
     ret = client.predict(feed_dict={"image": image}, fetch=["res"])
     print(ret)
-    #print(ret)

deploy/pdserving/web_service.py

@@ -56,11 +56,9 @@ class DetOp(Op):
 
         det_img = self.det_preprocess(self.im)
         _, self.new_h, self.new_w = det_img.shape
-        print("det image shape", det_img.shape)
         return {"x": det_img[np.newaxis, :].copy()}, False, None, ""
 
     def postprocess(self, input_dicts, fetch_dict, log_id):
-        print("input_dicts: ", input_dicts)
         det_out = fetch_dict["save_infer_model/scale_0.tmp_1"]
         ratio_list = [
             float(self.new_h) / self.ori_h, float(self.new_w) / self.ori_w
@@ -69,7 +67,6 @@ class DetOp(Op):
         dt_boxes = self.filter_func(dt_boxes_list[0], [self.ori_h, self.ori_w])
         out_dict = {"dt_boxes": dt_boxes, "image": self.im}
 
-        print("out dict", out_dict["dt_boxes"])
         return out_dict, None, ""
 
 
@@ -89,29 +86,67 @@ class RecOp(Op):
         feed_list = []
         img_list = []
         max_wh_ratio = 0
-        for i, dtbox in enumerate(dt_boxes):
-            boximg = self.get_rotate_crop_image(im, dt_boxes[i])
-            img_list.append(boximg)
-            h, w = boximg.shape[0:2]
-            wh_ratio = w * 1.0 / h
-            max_wh_ratio = max(max_wh_ratio, wh_ratio)
-        _, w, h = self.ocr_reader.resize_norm_img(img_list[0],
-                                                  max_wh_ratio).shape
-
-        imgs = np.zeros((len(img_list), 3, w, h)).astype('float32')
-        for id, img in enumerate(img_list):
-            norm_img = self.ocr_reader.resize_norm_img(img, max_wh_ratio)
-            imgs[id] = norm_img
-        print("rec image shape", imgs.shape)
-        feed = {"x": imgs.copy()}
-        return feed, False, None, ""
-
-    def postprocess(self, input_dicts, fetch_dict, log_id):
-        rec_res = self.ocr_reader.postprocess(fetch_dict, with_score=True)
-        res_lst = []
-        for res in rec_res:
-            res_lst.append(res[0])
-        res = {"res": str(res_lst)}
+        ## Many mini-batchs, the type of feed_data is list.
+        max_batch_size = 6  # len(dt_boxes)
+
+        # If max_batch_size is 0, skipping predict stage
+        if max_batch_size == 0:
+            return {}, True, None, ""
+        boxes_size = len(dt_boxes)
+        batch_size = boxes_size // max_batch_size
+        rem = boxes_size % max_batch_size
+        #_LOGGER.info("max_batch_len:{}, batch_size:{}, rem:{}, boxes_size:{}".format(max_batch_size, batch_size, rem, boxes_size))
+        for bt_idx in range(0, batch_size + 1):
+            imgs = None
+            boxes_num_in_one_batch = 0
+            if bt_idx == batch_size:
+                if rem == 0:
+                    continue
+                else:
+                    boxes_num_in_one_batch = rem
+            elif bt_idx < batch_size:
+                boxes_num_in_one_batch = max_batch_size
+            else:
+                _LOGGER.error("batch_size error, bt_idx={}, batch_size={}".
+                              format(bt_idx, batch_size))
+                break
+
+            start = bt_idx * max_batch_size
+            end = start + boxes_num_in_one_batch
+            img_list = []
+            for box_idx in range(start, end):
+                boximg = self.get_rotate_crop_image(im, dt_boxes[box_idx])
+                img_list.append(boximg)
+                h, w = boximg.shape[0:2]
+                wh_ratio = w * 1.0 / h
+                max_wh_ratio = max(max_wh_ratio, wh_ratio)
+            _, w, h = self.ocr_reader.resize_norm_img(img_list[0],
+                                                      max_wh_ratio).shape
+
+            imgs = np.zeros((boxes_num_in_one_batch, 3, w, h)).astype('float32')
+            for id, img in enumerate(img_list):
+                norm_img = self.ocr_reader.resize_norm_img(img, max_wh_ratio)
+                imgs[id] = norm_img
+            feed = {"x": imgs.copy()}
+            feed_list.append(feed)
+
+        return feed_list, False, None, ""
+    def postprocess(self, input_dicts, fetch_data, log_id):
+        res_list = []
+        if isinstance(fetch_data, dict):
+            if len(fetch_data) > 0:
+                rec_batch_res = self.ocr_reader.postprocess(
+                    fetch_data, with_score=True)
+                for res in rec_batch_res:
+                    res_list.append(res[0])
+        elif isinstance(fetch_data, list):
+            for one_batch in fetch_data:
+                one_batch_res = self.ocr_reader.postprocess(
+                    one_batch, with_score=True)
+                for res in one_batch_res:
+                    res_list.append(res[0])
+
+        res = {"res": str(res_list)}
         return res, None, ""