sRecognition Serving: support det and rec pipeline

deploy/paddleserving/recognition/config.yml

@@ -10,7 +10,7 @@ dag:
     #op资源类型, True, 为线程模型；False，为进程模型
     is_thread_op: False
 op:
-    recog:
+    rec:
         #并发数，is_thread_op=True时，为线程并发；否则为进程并发
         concurrency: 1
 
@@ -30,4 +30,14 @@ op:
             client_type: local_predictor
 
             #Fetch结果列表，以client_config中fetch_var的alias_name为准
-            fetch_list: ["features"]
+            fetch_list: ["features"]
\ No newline at end of file
+            
+    det:
+        concurrency: 1
+        local_service_conf:
+            client_type: local_predictor
+            device_type: 1
+            devices: '0'
+            fetch_list:
+            - save_infer_model/scale_0.tmp_1
+            model_config: ../../models/ppyolov2_r50vd_dcn_mainbody_v1.0_serving/
\ No newline at end of file

deploy/paddleserving/recognition/label_list.txt

+foreground
+background
\ No newline at end of file

deploy/paddleserving/recognition/pipeline_http_client.py

@@ -9,13 +9,13 @@ def cv2_to_base64(image):
     return base64.b64encode(image).decode('utf8')
 
 if __name__ == "__main__":
-    url = "http://127.0.0.1:18081/recog_service/prediction"
+    url = "http://127.0.0.1:18081/recognition/prediction"
 
     with open(os.path.join(".",  imgpath), 'rb') as file:
         image_data1 = file.read()
     image = cv2_to_base64(image_data1)
     data = {"key": ["image"], "value": [image]}
 
-    for i in range(5):
+    for i in range(1):
         r = requests.post(url=url, data=json.dumps(data))
         print(r.json())

deploy/paddleserving/recognition/pipeline_rpc_client.py

@@ -30,5 +30,5 @@ if __name__ == "__main__":
     image = cv2_to_base64(image_data)
 
     for i in range(1):
-        ret = client.predict(feed_dict={"image": image}, fetch=["label", "dist"])
+        ret = client.predict(feed_dict={"image": image}, fetch=["result"])
         print(ret)

deploy/paddleserving/recognition/recognition_web_service.py

@@ -11,20 +11,81 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import sys
-from paddle_serving_app.reader import Sequential, URL2Image, Resize, CenterCrop, RGB2BGR, Transpose, Div, Normalize, Base64ToImage
-try:
-    from paddle_serving_server_gpu.web_service import WebService, Op
-except ImportError:
-    from paddle_serving_server.web_service import WebService, Op
+from paddle_serving_server.web_service import WebService, Op
 import logging
 import numpy as np
-import base64, cv2
+import sys
+import cv2
+from paddle_serving_app.reader import *
+import base64
 import os
 import faiss
 import pickle
+import json
 
-class RecogOp(Op):
+class DetOp(Op):
+    def init_op(self):
+        self.img_preprocess = Sequential([
+            BGR2RGB(), Div(255.0),
+            Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225], False),
+            Resize((640, 640)), Transpose((2, 0, 1))
+        ])
+
+        self.img_postprocess = RCNNPostprocess("label_list.txt", "output")
+        self.threshold = 0.2
+        self.max_det_results = 5
+
+    def generate_scale(self, im):
+        """
+        Args:
+            im (np.ndarray): image (np.ndarray)
+        Returns:
+            im_scale_x: the resize ratio of X
+            im_scale_y: the resize ratio of Y
+        """
+        target_size = [640, 640]
+        origin_shape = im.shape[:2]
+        resize_h, resize_w = target_size
+        im_scale_y = resize_h / float(origin_shape[0])
+        im_scale_x = resize_w / float(origin_shape[1])
+        return im_scale_y, im_scale_x
+
+    def preprocess(self, input_dicts, data_id, log_id):
+        (_, input_dict), = input_dicts.items()
+        imgs = []
+        raw_imgs = []
+        for key in input_dict.keys():
+            data = base64.b64decode(input_dict[key].encode('utf8'))
+            raw_imgs.append(data)
+            data = np.fromstring(data, np.uint8)
+            raw_im = cv2.imdecode(data, cv2.IMREAD_COLOR)
+
+            im_scale_y, im_scale_x = self.generate_scale(raw_im)
+            im = self.img_preprocess(raw_im)
+            
+            imgs.append({
+              "image": im[np.newaxis, :],
+              "im_shape": np.array(list(im.shape[1:])).reshape(-1)[np.newaxis,:],
+              "scale_factor": np.array([im_scale_y, im_scale_x]).astype('float32'),
+            })
+        self.raw_img = raw_imgs
+
+        feed_dict = {
+            "image":        np.concatenate([x["image"] for x in imgs], axis=0),
+            "im_shape":     np.concatenate([x["im_shape"] for x in imgs], axis=0),
+            "scale_factor": np.concatenate([x["scale_factor"] for x in imgs], axis=0)
+        }
+        return feed_dict, False,  None,  ""
+
+    def postprocess(self, input_dicts, fetch_dict, log_id):
+        boxes = self.img_postprocess(fetch_dict, visualize=False)
+        boxes.sort(key = lambda x: x["score"], reverse = True)
+        boxes = filter(lambda x: x["score"] >= self.threshold, boxes[:self.max_det_results])
+        result = json.dumps(list(boxes))
+        res_dict = {"bbox_result": result, "image": self.raw_img}    
+        return res_dict,  None,  ""
+
+class RecOp(Op):
     def init_op(self):
         self.seq = Sequential([
             Resize(256), CenterCrop(224), RGB2BGR(), Transpose((2, 0, 1)),
@@ -32,7 +93,6 @@ class RecogOp(Op):
                                 True)
         ])
 
-        #load index; and return top1
         index_dir = "../../recognition_demo_data_v1.1/gallery_product/index"
         assert os.path.exists(os.path.join(
             index_dir, "vector.index")), "vector.index not found ..."
@@ -45,35 +105,82 @@ class RecogOp(Op):
         with open(os.path.join(index_dir, "id_map.pkl"), "rb") as fd:
             self.id_map = pickle.load(fd)
 
+        self.rec_nms_thresold = 0.05
+        self.rec_score_thres = 0.5
+
     def preprocess(self, input_dicts, data_id, log_id):
         (_, input_dict), = input_dicts.items()
-        batch_size = len(input_dict.keys())
+        raw_img = input_dict["image"][0]
+        data = np.frombuffer(raw_img, np.uint8)
+        origin_img = cv2.imdecode(data, cv2.IMREAD_COLOR)
+        dt_boxes = input_dict["bbox_result"]
+        boxes = json.loads(dt_boxes)
+        boxes.append({"category_id": 0,
+                      "score": 1.0,
+                      "bbox": [0, 0, origin_img.shape[1], origin_img.shape[0]]
+                     })
+        self.det_boxes = boxes
+
+        #construct batch images for rec
         imgs = []
-        for key in input_dict.keys():
-            data = base64.b64decode(input_dict[key].encode('utf8'))
-            data = np.fromstring(data, np.uint8)
-            im = cv2.imdecode(data, cv2.IMREAD_COLOR)
+        for box in boxes:
+            box = [int(x) for x in box["bbox"]]
+            im = origin_img[box[1]: box[1] + box[3], box[0]: box[0] + box[2]].copy()
             img = self.seq(im)
             imgs.append(img[np.newaxis, :].copy())
+
         input_imgs = np.concatenate(imgs, axis=0)
-        return {"x": input_imgs}, False, None, ""
+        return {"x": input_imgs},  False,  None,  ""
+
+    def nms_to_rec_results(self, results, thresh = 0.1):
+        filtered_results = []
+        x1 = np.array([r["bbox"][0] for r in results]).astype("float32")
+        y1 = np.array([r["bbox"][1] for r in results]).astype("float32")
+        x2 = np.array([r["bbox"][2] for r in results]).astype("float32")
+        y2 = np.array([r["bbox"][3] for r in results]).astype("float32")
+        scores = np.array([r["rec_scores"] for r in results])
+
+        areas = (x2 - x1 + 1) * (y2 - y1 + 1)
+        order = scores.argsort()[::-1]
+        while order.size > 0:
+            i = order[0]
+            xx1 = np.maximum(x1[i], x1[order[1:]])
+            yy1 = np.maximum(y1[i], y1[order[1:]])
+            xx2 = np.minimum(x2[i], x2[order[1:]])
+            yy2 = np.minimum(y2[i], y2[order[1:]])
+
+            w = np.maximum(0.0, xx2 - xx1 + 1)
+            h = np.maximum(0.0, yy2 - yy1 + 1)
+            inter = w * h
+            ovr = inter / (areas[i] + areas[order[1:]] - inter)
+            inds = np.where(ovr <= thresh)[0]
+            order = order[inds + 1]
+            filtered_results.append(results[i])
+        return filtered_results
 
     def postprocess(self, input_dicts, fetch_dict, log_id):
         score_list = fetch_dict["features"]
-        
-        return_top_k = 1
-        scores, docs = self.searcher.search(score_list,  return_top_k)  
+        scores, docs = self.searcher.search(score_list,  1) 
 
-        result = {}
-        result["label"] = self.id_map[docs[0][0]].split()[1]
-        result["dist"] = str(scores[0][0])
-        return result, None, ""
+        results = []
+        for i in range(scores.shape[0]):
+            pred = {}
+            if scores[i][0] >= self.rec_score_thres:
+                pred["bbox"] = self.det_boxes[i]["bbox"]
+                pred["rec_docs"] = self.id_map[docs[i][0]].split()[1]
+                pred["rec_scores"] = scores[i][0]
+                results.append(pred)
+        
+        #do nms
+        results = self.nms_to_rec_results(results, self.rec_nms_thresold)
+        return {"result": str(results)}, None, ""
 
-class ProductRecognitionService(WebService):
+class RecognitionService(WebService):
     def get_pipeline_response(self, read_op):
-        image_op = RecogOp(name="recog", input_ops=[read_op])
-        return image_op
+        det_op = DetOp(name="det", input_ops=[read_op])
+        rec_op = RecOp(name="rec", input_ops=[det_op])
+        return rec_op
 
-uci_service = ProductRecognitionService(name="recog_service")
-uci_service.prepare_pipeline_config("config.yml")
-uci_service.run_service()
+product_recog_service = RecognitionService(name="recognition")
+product_recog_service.prepare_pipeline_config("config.yml")
+product_recog_service.run_service()