recognition_web_service.py

# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from paddle_serving_server.web_service import WebService, Op
import logging
import numpy as np
import sys
import cv2
from paddle_serving_app.reader import *
import base64
import os
import faiss
import pickle
import json

class DetOp(Op):
    def init_op(self):
        self.img_preprocess = Sequential([
            BGR2RGB(), Div(255.0),
            Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225], False),
            Resize((640, 640)), Transpose((2, 0, 1))
        ])

        self.img_postprocess = RCNNPostprocess("label_list.txt", "output")
        self.threshold = 0.2
        self.max_det_results = 5

    def generate_scale(self, im):
        """
        Args:
            im (np.ndarray): image (np.ndarray)
        Returns:
            im_scale_x: the resize ratio of X
            im_scale_y: the resize ratio of Y
        """
        target_size = [640, 640]
        origin_shape = im.shape[:2]
        resize_h, resize_w = target_size
        im_scale_y = resize_h / float(origin_shape[0])
        im_scale_x = resize_w / float(origin_shape[1])
        return im_scale_y, im_scale_x

    def preprocess(self, input_dicts, data_id, log_id):
        (_, input_dict), = input_dicts.items()
        imgs = []
        raw_imgs = []
        for key in input_dict.keys():
            data = base64.b64decode(input_dict[key].encode('utf8'))
            raw_imgs.append(data)
            data = np.fromstring(data, np.uint8)
            raw_im = cv2.imdecode(data, cv2.IMREAD_COLOR)

            im_scale_y, im_scale_x = self.generate_scale(raw_im)
            im = self.img_preprocess(raw_im)
            
            imgs.append({
              "image": im[np.newaxis, :],
              "im_shape": np.array(list(im.shape[1:])).reshape(-1)[np.newaxis,:],
              "scale_factor": np.array([im_scale_y, im_scale_x]).astype('float32'),
            })
        self.raw_img = raw_imgs

        feed_dict = {
            "image":        np.concatenate([x["image"] for x in imgs], axis=0),
            "im_shape":     np.concatenate([x["im_shape"] for x in imgs], axis=0),
            "scale_factor": np.concatenate([x["scale_factor"] for x in imgs], axis=0)
        }
        return feed_dict, False,  None,  ""

    def postprocess(self, input_dicts, fetch_dict, log_id):
        boxes = self.img_postprocess(fetch_dict, visualize=False)
        boxes.sort(key = lambda x: x["score"], reverse = True)
        boxes = filter(lambda x: x["score"] >= self.threshold, boxes[:self.max_det_results])
        boxes = list(boxes)
        for i in range(len(boxes)):
            boxes[i]["bbox"][2] += boxes[i]["bbox"][0] - 1
            boxes[i]["bbox"][3] += boxes[i]["bbox"][1] - 1
        result = json.dumps(boxes)
        res_dict = {"bbox_result": result, "image": self.raw_img}
        return res_dict,  None,  ""

class RecOp(Op):
    def init_op(self):
        self.seq = Sequential([
            BGR2RGB(), Resize((224, 224)), 
            Div(255), Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225],
                                False), Transpose((2, 0, 1))
        ])

        index_dir = "../../recognition_demo_data_v1.1/gallery_product/index"
        assert os.path.exists(os.path.join(
            index_dir, "vector.index")), "vector.index not found ..."
        assert os.path.exists(os.path.join(
            index_dir, "id_map.pkl")), "id_map.pkl not found ... "
        
        self.searcher = faiss.read_index(
            os.path.join(index_dir, "vector.index"))
                
        with open(os.path.join(index_dir, "id_map.pkl"), "rb") as fd:
            self.id_map = pickle.load(fd)

        self.rec_nms_thresold = 0.05
        self.rec_score_thres = 0.5
        self.feature_normalize = True
        self.return_k = 1

    def preprocess(self, input_dicts, data_id, log_id):
        (_, input_dict), = input_dicts.items()
        raw_img = input_dict["image"][0]
        data = np.frombuffer(raw_img, np.uint8)
        origin_img = cv2.imdecode(data, cv2.IMREAD_COLOR)
        dt_boxes = input_dict["bbox_result"]
        boxes = json.loads(dt_boxes)
        boxes.append({"category_id": 0,
                      "score": 1.0,
                      "bbox": [0, 0, origin_img.shape[1], origin_img.shape[0]]
                     })
        self.det_boxes = boxes

        #construct batch images for rec
        imgs = []
        for box in boxes:
            box = [int(x) for x in box["bbox"]]
            im = origin_img[box[1]: box[3], box[0]: box[2]].copy()
            img = self.seq(im)
            imgs.append(img[np.newaxis, :].copy())

        input_imgs = np.concatenate(imgs, axis=0)
        return {"x": input_imgs},  False,  None,  ""

    def nms_to_rec_results(self, results, thresh = 0.1):
        filtered_results = []
        x1 = np.array([r["bbox"][0] for r in results]).astype("float32")
        y1 = np.array([r["bbox"][1] for r in results]).astype("float32")
        x2 = np.array([r["bbox"][2] for r in results]).astype("float32")
        y2 = np.array([r["bbox"][3] for r in results]).astype("float32")
        scores = np.array([r["rec_scores"] for r in results])

        areas = (x2 - x1 + 1) * (y2 - y1 + 1)
        order = scores.argsort()[::-1]
        while order.size > 0:
            i = order[0]
            xx1 = np.maximum(x1[i], x1[order[1:]])
            yy1 = np.maximum(y1[i], y1[order[1:]])
            xx2 = np.minimum(x2[i], x2[order[1:]])
            yy2 = np.minimum(y2[i], y2[order[1:]])

            w = np.maximum(0.0, xx2 - xx1 + 1)
            h = np.maximum(0.0, yy2 - yy1 + 1)
            inter = w * h
            ovr = inter / (areas[i] + areas[order[1:]] - inter)
            inds = np.where(ovr <= thresh)[0]
            order = order[inds + 1]
            filtered_results.append(results[i])
        return filtered_results

    def postprocess(self, input_dicts, fetch_dict, log_id):
        batch_features = fetch_dict["features"]

        if self.feature_normalize:
            feas_norm = np.sqrt(
                np.sum(np.square(batch_features), axis=1, keepdims=True))
            batch_features = np.divide(batch_features, feas_norm)

        scores, docs = self.searcher.search(batch_features,  self.return_k)

        results = []
        for i in range(scores.shape[0]):
            pred = {}
            if scores[i][0] >= self.rec_score_thres:
                pred["bbox"] = [int(x) for x in self.det_boxes[i]["bbox"]]
                pred["rec_docs"] = self.id_map[docs[i][0]].split()[1]
                pred["rec_scores"] = scores[i][0]
                results.append(pred)
        
        #do nms
        results = self.nms_to_rec_results(results, self.rec_nms_thresold)
        return {"result": str(results)}, None, ""

class RecognitionService(WebService):
    def get_pipeline_response(self, read_op):
        det_op = DetOp(name="det", input_ops=[read_op])
        rec_op = RecOp(name="rec", input_ops=[det_op])
        return rec_op

product_recog_service = RecognitionService(name="recognition")
product_recog_service.prepare_pipeline_config("config.yml")
product_recog_service.run_service()