# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import time import sys import argparse import numpy as np import cv2 import yaml import paddle from paddle.inference import create_predictor from paddle.io import DataLoader from imagenet_reader import ImageNetDataset def argsparser(): """ argsparser func """ parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--model_path", type=str, default="./MobileNetV1_infer", help="model directory") parser.add_argument( "--model_filename", type=str, default="inference.pdmodel", help="model file name") parser.add_argument( "--params_filename", type=str, default="inference.pdiparams", help="params file name") parser.add_argument("--batch_size", type=int, default=1) parser.add_argument("--img_size", type=int, default=224) parser.add_argument("--resize_size", type=int, default=256) parser.add_argument( "--data_path", type=str, default="./dataset/ILSVRC2012/") parser.add_argument( "--use_gpu", type=bool, default=False, help="Whether to use gpu") parser.add_argument( "--use_trt", type=bool, default=False, help="Whether to use tensorrt") parser.add_argument( "--use_mkldnn", type=bool, default=False, help="Whether to use mkldnn") parser.add_argument( "--cpu_num_threads", type=int, default=10, help="Number of cpu threads") parser.add_argument( "--use_fp16", type=bool, default=False, help="Whether to use fp16") parser.add_argument( "--use_int8", type=bool, default=False, help="Whether to use int8") parser.add_argument("--gpu_mem", type=int, default=8000, help="GPU memory") parser.add_argument("--ir_optim", type=bool, default=True) parser.add_argument( "--use_dynamic_shape", type=bool, default=True, help="Whether use dynamic shape or not.") return parser def eval_reader(data_dir, batch_size, crop_size, resize_size): """ eval reader func """ val_reader = ImageNetDataset( mode="val", data_dir=data_dir, crop_size=crop_size, resize_size=resize_size) val_loader = DataLoader( val_reader, batch_size=args.batch_size, shuffle=False, drop_last=False, num_workers=0) return val_loader class Predictor(object): """ Paddle Inference Predictor class """ def __init__(self): # HALF precission predict only work when using tensorrt if args.use_fp16 is True: assert args.use_trt is True self.rerun_flag = False self.paddle_predictor = self._create_paddle_predictor() input_names = self.paddle_predictor.get_input_names() self.input_tensor = self.paddle_predictor.get_input_handle(input_names[ 0]) output_names = self.paddle_predictor.get_output_names() self.output_tensor = self.paddle_predictor.get_output_handle( output_names[0]) def _create_paddle_predictor(self): inference_model_dir = args.model_path model_file = os.path.join(inference_model_dir, args.model_filename) params_file = os.path.join(inference_model_dir, args.params_filename) config = paddle.inference.Config(model_file, params_file) precision = paddle.inference.Config.Precision.Float32 if args.use_int8: precision = paddle.inference.Config.Precision.Int8 elif args.use_fp16: precision = paddle.inference.Config.Precision.Half if args.use_gpu: config.enable_use_gpu(args.gpu_mem, 0) else: config.disable_gpu() config.set_cpu_math_library_num_threads(args.cpu_num_threads) config.switch_ir_optim() if args.use_mkldnn: config.enable_mkldnn() if args.use_int8: config.enable_mkldnn_int8( {"conv2d", "depthwise_conv2d", "transpose2", "pool2d"}) config.switch_ir_optim(args.ir_optim) # default true if args.use_trt: config.enable_tensorrt_engine( precision_mode=precision, max_batch_size=args.batch_size, workspace_size=1 << 30, min_subgraph_size=30, use_static=True, use_calib_mode=False, ) if args.use_dynamic_shape: dynamic_shape_file = os.path.join(inference_model_dir, "dynamic_shape.txt") if os.path.exists(dynamic_shape_file): config.enable_tuned_tensorrt_dynamic_shape( dynamic_shape_file, True) print("trt set dynamic shape done!") else: config.collect_shape_range_info(dynamic_shape_file) print("Start collect dynamic shape...") self.rerun_flag = True config.enable_memory_optim() predictor = create_predictor(config) return predictor def eval(self): """ eval func """ if os.path.exists(args.data_path): val_loader = eval_reader( args.data_path, batch_size=args.batch_size, crop_size=args.img_size, resize_size=args.resize_size) else: image = np.ones( (1, 3, args.img_size, args.img_size)).astype(np.float32) label = None val_loader = [[image, label]] results = [] input_names = self.paddle_predictor.get_input_names() input_tensor = self.paddle_predictor.get_input_handle(input_names[0]) output_names = self.paddle_predictor.get_output_names() output_tensor = self.paddle_predictor.get_output_handle(output_names[0]) predict_time = 0.0 time_min = float("inf") time_max = float("-inf") sample_nums = len(val_loader) for batch_id, (image, label) in enumerate(val_loader): image = np.array(image) input_tensor.copy_from_cpu(image) start_time = time.time() self.paddle_predictor.run() batch_output = output_tensor.copy_to_cpu() end_time = time.time() timed = end_time - start_time time_min = min(time_min, timed) time_max = max(time_max, timed) predict_time += timed if self.rerun_flag: return sort_array = batch_output.argsort(axis=1) top_1_pred = sort_array[:, -1:][:, ::-1] if label is None: results.append(top_1_pred) break label = np.array(label) top_1 = np.mean(label == top_1_pred) top_5_pred = sort_array[:, -5:][:, ::-1] acc_num = 0 for i, _ in enumerate(label): if label[i][0] in top_5_pred[i]: acc_num += 1 top_5 = float(acc_num) / len(label) results.append([top_1, top_5]) if batch_id % 100 == 0: print("Eval iter:", batch_id) sys.stdout.flush() result = np.mean(np.array(results), axis=0) fp_message = "FP16" if args.use_fp16 else "FP32" fp_message = "INT8" if args.use_int8 else fp_message print_msg = "Paddle" if args.use_trt: print_msg = "using TensorRT" elif args.use_mkldnn: print_msg = "using MKLDNN" time_avg = predict_time / sample_nums print( "[Benchmark]{}\t{}\tbatch size: {}.Inference time(ms): min={}, max={}, avg={}". format( print_msg, fp_message, args.batch_size, round(time_min * 1000, 2), round(time_max * 1000, 1), round(time_avg * 1000, 1), )) print("[Benchmark] Evaluation acc result: {}".format(result[0])) sys.stdout.flush() if __name__ == "__main__": parser = argsparser() args = parser.parse_args() predictor = Predictor() predictor.eval() if predictor.rerun_flag: print( "***** Collect dynamic shape done, Please rerun the program to get correct results. *****" )