Merge pull request #806 from FredHuang16/patch-9

Separate indexing construction and retrieval

deploy/configs/build_logo.yaml

+Global:
+  rec_inference_model_dir: "./logo/model/"
+  batch_size: 1
+  use_gpu: True
+  enable_mkldnn: True
+  cpu_num_threads: 100
+  enable_benchmark: True
+  use_fp16: False
+  ir_optim: True
+  use_tensorrt: False
+  gpu_mem: 8000
+  enable_profile: False
+
+RecPreProcess:
+  transform_ops:
+    - ResizeImage:
+        size: 224
+    - NormalizeImage:
+        scale: 0.00392157
+        mean: [0.485, 0.456, 0.406]
+        std: [0.229, 0.224, 0.225]
+        order: ''
+    - ToCHWImage:
+
+RecPostProcess: null
+
+# indexing engine config
+IndexProcess:
+  index_path: "./logo/logo_index/"
+  image_root: "./logo/dataset/"
+  data_file:  "./logo/logo_gallery_demo.txt"
+  delimiter: "\t"
+  dist_type: "IP"
+  pq_size: 100
+  embedding_size: 512

deploy/configs/inference_logo.yaml

+Global:
+  infer_imgs: "./logo/demo/logo_APK.jpg"
+  det_inference_model_dir: "./logo/det/"
+  rec_inference_model_dir: "./logo/rec/"
+  batch_size: 1
+  image_shape: [3, 640, 640]
+  threshold: 0.5
+  max_det_results: 1
+  labe_list:
+  - foreground
+
+  # inference engine config
+  use_gpu: True
+  enable_mkldnn: True
+  cpu_num_threads: 100
+  enable_benchmark: True
+  use_fp16: False
+  ir_optim: True
+  use_tensorrt: False
+  gpu_mem: 8000
+  enable_profile: False
+
+DetPreProcess:
+  transform_ops:
+    - DetResize:
+        interp: 2
+        keep_ratio: false
+        target_size: [640, 640]
+    - DetNormalizeImage:
+        is_scale: true
+        mean: [0.485, 0.456, 0.406]
+        std: [0.229, 0.224, 0.225]
+    - DetPermute: {}
+DetPostProcess: {}
+
+RecPreProcess:
+  transform_ops:
+    - ResizeImage:
+        size: 224
+    - NormalizeImage:
+        scale: 0.00392157
+        mean: [0.485, 0.456, 0.406]
+        std: [0.229, 0.224, 0.225]
+        order: ''
+    - ToCHWImage:
+
+RecPostProcess: null
+
+# indexing engine config
+IndexProcess:
+  index_path: "./logo_index/"
+  search_budget: 100
+  return_k: 10
+  dist_type: "IP"

deploy/python/build_gallery.py

+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+import sys
+
+__dir__ = os.path.dirname(os.path.abspath(__file__))
+sys.path.append(os.path.abspath(os.path.join(__dir__, '../')))
+
+import copy
+import cv2
+import numpy as np
+from tqdm import tqdm
+
+from python.predict_rec import RecPredictor
+from vector_search import Graph_Index
+
+from utils import logger
+from utils import config
+
+
+def split_datafile(data_file, image_root, delimiter="\t"):
+    '''
+        data_file: image path and info, which can be splitted by spacer 
+        image_root: image path root
+        delimiter: delimiter 
+    '''
+    gallery_images = []
+    gallery_docs = []
+    with open(data_file) as f:
+        lines = f.readlines()
+        for i, line in enumerate(lines):
+            line = line.strip().split(delimiter)
+            image_file = os.path.join(image_root, line[0])
+            image_doc = line[1] 
+            gallery_images.append(image_file)
+            gallery_docs.append(image_doc)
+
+    return gallery_images, gallery_docs
+
+
+class GalleryBuilder(object):
+    def __init__(self, config):
+
+        self.config = config
+        self.rec_predictor = RecPredictor(config)
+        assert 'IndexProcess' in config.keys(), "Index config not found ... "
+        self.build(config['IndexProcess'])
+
+
+    def build(self, config):
+        '''
+            build index from scratch
+        '''
+        gallery_images, gallery_docs = split_datafile(config['data_file'], 
+            config['image_root'], config['delimiter'])
+
+        # extract gallery features
+        gallery_features = np.zeros([len(gallery_images), 
+            config['embedding_size']], dtype=np.float32)
+
+        for i, image_file in enumerate(tqdm(gallery_images)):
+            img = cv2.imread(image_file)[:, :, ::-1]
+            rec_feat = self.rec_predictor.predict(img)
+            gallery_features[i,:] = rec_feat
+
+        # train index 
+        self.Searcher = Graph_Index(dist_type=config['dist_type']) 
+        self.Searcher.build(gallery_vectors=gallery_features, gallery_docs=gallery_docs, 
+            pq_size=config['pq_size'], index_path=config['index_path'])
+            
+
+def main(config):
+    system_builder = GalleryBuilder(config)
+    return
+
+
+if __name__ == "__main__":
+    args = config.parse_args()
+    config = config.get_config(args.config, overrides=args.override, show=True)
+    main(config)

deploy/python/predict_system.py

@@ -29,23 +29,6 @@ from utils import logger
 from utils import config
 from utils.get_image_list import get_image_list
 
-
-def split_datafile(data_file, image_root):
-    gallery_images = []
-    gallery_docs = []
-    with open(data_file) as f:
-        lines = f.readlines()
-        for i, line in enumerate(lines):
-            line = line.strip().split("\t")
-            if line[0] == 'image_id':
-                 continue
-            image_file = os.path.join(image_root, line[3])
-            image_doc = line[1]
-            gallery_images.append(image_file)
-            gallery_docs.append(image_doc)
-    return gallery_images, gallery_docs
-
-
 class SystemPredictor(object):
     def __init__(self, config):
 
@@ -54,48 +37,30 @@ class SystemPredictor(object):
         self.det_predictor = DetPredictor(config)
 
         assert 'IndexProcess' in config.keys(), "Index config not found ... "
-        self.indexer(config['IndexProcess'])
-        self.return_k = self.config['IndexProcess']['infer']['return_k']
-        self.search_budget = self.config['IndexProcess']['infer']['search_budget']
-    
-    def indexer(self, config):
-        if 'build' in config.keys() and config['build']['enable']:  # build the index from scratch    
-            with open(config['build']['data_file']) as f:
-                lines = f.readlines()
-            gallery_images, gallery_docs = split_datafile(config['build']['data_file'], config['build']['image_root'])
-            # extract gallery features
-            gallery_features = np.zeros([len(gallery_images), config['build']['embedding_size']], dtype=np.float32)
-            for i, image_file in enumerate(gallery_images):
-                img = cv2.imread(image_file)[:, :, ::-1]
-                rec_feat = self.rec_predictor.predict(img)
-                gallery_features[i,:] = rec_feat
-            # train index 
-            self.Searcher = Graph_Index(dist_type=config['build']['dist_type']) 
-            self.Searcher.build(gallery_vectors=gallery_features, gallery_docs=gallery_docs, 
-                pq_size=config['build']['pq_size'], index_path=config['build']['index_path'])
-            
-        else:   # load local index
-            self.Searcher = Graph_Index(dist_type=config['build']['dist_type']) 
-            self.Searcher.load(config['infer']['index_path'])
+        self.return_k = self.config['IndexProcess']['return_k']
+        self.search_budget = self.config['IndexProcess']['search_budget']
+
+        self.Searcher = Graph_Index(dist_type=config['IndexProcess']['dist_type']) 
+        self.Searcher.load(config['IndexProcess']['index_path'])
 
     def predict(self, img):
         output = []
         results = self.det_predictor.predict(img)
         for result in results:
+            preds = {}
             xmin, ymin, xmax, ymax = result["bbox"].astype("int")
             crop_img = img[xmin:xmax, ymin:ymax, :].copy()
             rec_results = self.rec_predictor.predict(crop_img)
-            result["feature"] = rec_results
-
+            #preds["feature"] = rec_results
+            preds["bbox"] = [xmin, ymin, xmax, ymax]
             scores, docs = self.Searcher.search(query=rec_results, return_k=self.return_k, search_budget=self.search_budget)
-            result["ret_docs"] = docs
-            result["ret_scores"] = scores
+            preds["rec_docs"] = docs
+            preds["rec_scores"] = scores
 
-            output.append(result)
+            output.append(preds)
         return output
             
 
-
 def main(config):
     system_predictor = SystemPredictor(config)
     image_list = get_image_list(config["Global"]["infer_imgs"])
@@ -104,7 +69,7 @@ def main(config):
     for idx, image_file in enumerate(image_list):
         img = cv2.imread(image_file)[:, :, ::-1]
         output = system_predictor.predict(img)
-        #print(output)
+        print(output)
     return
 
 

deploy/shell/predict.sh

@@ -7,5 +7,9 @@ python3.7 python/predict_cls.py -c configs/inference_cls.yaml
 # detection
 # python3.7 python/predict_det.py -c configs/inference_rec.yaml
 
-# mainbody detection + feature extractor + retrieval
-# python3.7 python/predict_system.py -c configs/inference_rec.yaml 
+
+# build system
+#python3.7 python/build_gallery.py -c configs/build_logo.yaml 
+
+# inference system
+# python3.7 python/predict_system.py -c configs/inference_logo.yaml