From ebbc54319322d505a589a2e0baf7bdec0a0f4c37 Mon Sep 17 00:00:00 2001 From: Guanghua Yu <742925032@qq.com> Date: Mon, 15 Aug 2022 13:38:39 +0800 Subject: [PATCH] update YOLO ptq dataloader (#1340) --- .../pytorch_yolov5/post_quant.py | 38 +++++-------------- .../pytorch_yolov6/post_quant.py | 38 +++++-------------- .../pytorch_yolov7/post_quant.py | 38 +++++-------------- 3 files changed, 30 insertions(+), 84 deletions(-) diff --git a/example/auto_compression/pytorch_yolov5/post_quant.py b/example/auto_compression/pytorch_yolov5/post_quant.py index a5316d29..84db4f98 100644 --- a/example/auto_compression/pytorch_yolov5/post_quant.py +++ b/example/auto_compression/pytorch_yolov5/post_quant.py @@ -17,13 +17,10 @@ import sys import numpy as np import argparse import paddle -from ppdet.core.workspace import load_config, merge_config -from ppdet.core.workspace import create -from ppdet.metrics import COCOMetric, VOCMetric -from paddleslim.auto_compression.config_helpers import load_config as load_slim_config -from paddleslim.quant import quant_post_static +from paddleslim.auto_compression.config_helpers import load_config from paddleslim.common import load_onnx_model - +from paddleslim.quant import quant_post_static +from dataset import COCOTrainDataset def argsparser(): parser = argparse.ArgumentParser(description=__doc__) @@ -49,32 +46,17 @@ def argsparser(): return parser -def reader_wrapper(reader, input_list): - def gen(): - for data in reader: - in_dict = {} - if isinstance(input_list, list): - for input_name in input_list: - in_dict[input_name] = data[input_name] - elif isinstance(input_list, dict): - for input_name in input_list.keys(): - in_dict[input_list[input_name]] = data[input_name] - yield in_dict - - return gen - - def main(): global global_config - all_config = load_slim_config(FLAGS.config_path) - assert "Global" in all_config, f"Key 'Global' not found in config file. \n{all_config}" + all_config = load_config(FLAGS.config_path) global_config = all_config["Global"] - reader_cfg = load_config(global_config['reader_config']) - train_loader = create('EvalReader')(reader_cfg['TrainDataset'], - reader_cfg['worker_num'], - return_list=True) - train_loader = reader_wrapper(train_loader, global_config['input_list']) + dataset = COCOTrainDataset( + dataset_dir=global_config['dataset_dir'], + image_dir=global_config['val_image_dir'], + anno_path=global_config['val_anno_path']) + train_loader = paddle.io.DataLoader( + dataset, batch_size=1, shuffle=True, drop_last=True, num_workers=0) place = paddle.CUDAPlace(0) if FLAGS.devices == 'gpu' else paddle.CPUPlace() exe = paddle.static.Executor(place) diff --git a/example/auto_compression/pytorch_yolov6/post_quant.py b/example/auto_compression/pytorch_yolov6/post_quant.py index a5316d29..84db4f98 100644 --- a/example/auto_compression/pytorch_yolov6/post_quant.py +++ b/example/auto_compression/pytorch_yolov6/post_quant.py @@ -17,13 +17,10 @@ import sys import numpy as np import argparse import paddle -from ppdet.core.workspace import load_config, merge_config -from ppdet.core.workspace import create -from ppdet.metrics import COCOMetric, VOCMetric -from paddleslim.auto_compression.config_helpers import load_config as load_slim_config -from paddleslim.quant import quant_post_static +from paddleslim.auto_compression.config_helpers import load_config from paddleslim.common import load_onnx_model - +from paddleslim.quant import quant_post_static +from dataset import COCOTrainDataset def argsparser(): parser = argparse.ArgumentParser(description=__doc__) @@ -49,32 +46,17 @@ def argsparser(): return parser -def reader_wrapper(reader, input_list): - def gen(): - for data in reader: - in_dict = {} - if isinstance(input_list, list): - for input_name in input_list: - in_dict[input_name] = data[input_name] - elif isinstance(input_list, dict): - for input_name in input_list.keys(): - in_dict[input_list[input_name]] = data[input_name] - yield in_dict - - return gen - - def main(): global global_config - all_config = load_slim_config(FLAGS.config_path) - assert "Global" in all_config, f"Key 'Global' not found in config file. \n{all_config}" + all_config = load_config(FLAGS.config_path) global_config = all_config["Global"] - reader_cfg = load_config(global_config['reader_config']) - train_loader = create('EvalReader')(reader_cfg['TrainDataset'], - reader_cfg['worker_num'], - return_list=True) - train_loader = reader_wrapper(train_loader, global_config['input_list']) + dataset = COCOTrainDataset( + dataset_dir=global_config['dataset_dir'], + image_dir=global_config['val_image_dir'], + anno_path=global_config['val_anno_path']) + train_loader = paddle.io.DataLoader( + dataset, batch_size=1, shuffle=True, drop_last=True, num_workers=0) place = paddle.CUDAPlace(0) if FLAGS.devices == 'gpu' else paddle.CPUPlace() exe = paddle.static.Executor(place) diff --git a/example/auto_compression/pytorch_yolov7/post_quant.py b/example/auto_compression/pytorch_yolov7/post_quant.py index a5316d29..84db4f98 100644 --- a/example/auto_compression/pytorch_yolov7/post_quant.py +++ b/example/auto_compression/pytorch_yolov7/post_quant.py @@ -17,13 +17,10 @@ import sys import numpy as np import argparse import paddle -from ppdet.core.workspace import load_config, merge_config -from ppdet.core.workspace import create -from ppdet.metrics import COCOMetric, VOCMetric -from paddleslim.auto_compression.config_helpers import load_config as load_slim_config -from paddleslim.quant import quant_post_static +from paddleslim.auto_compression.config_helpers import load_config from paddleslim.common import load_onnx_model - +from paddleslim.quant import quant_post_static +from dataset import COCOTrainDataset def argsparser(): parser = argparse.ArgumentParser(description=__doc__) @@ -49,32 +46,17 @@ def argsparser(): return parser -def reader_wrapper(reader, input_list): - def gen(): - for data in reader: - in_dict = {} - if isinstance(input_list, list): - for input_name in input_list: - in_dict[input_name] = data[input_name] - elif isinstance(input_list, dict): - for input_name in input_list.keys(): - in_dict[input_list[input_name]] = data[input_name] - yield in_dict - - return gen - - def main(): global global_config - all_config = load_slim_config(FLAGS.config_path) - assert "Global" in all_config, f"Key 'Global' not found in config file. \n{all_config}" + all_config = load_config(FLAGS.config_path) global_config = all_config["Global"] - reader_cfg = load_config(global_config['reader_config']) - train_loader = create('EvalReader')(reader_cfg['TrainDataset'], - reader_cfg['worker_num'], - return_list=True) - train_loader = reader_wrapper(train_loader, global_config['input_list']) + dataset = COCOTrainDataset( + dataset_dir=global_config['dataset_dir'], + image_dir=global_config['val_image_dir'], + anno_path=global_config['val_anno_path']) + train_loader = paddle.io.DataLoader( + dataset, batch_size=1, shuffle=True, drop_last=True, num_workers=0) place = paddle.CUDAPlace(0) if FLAGS.devices == 'gpu' else paddle.CPUPlace() exe = paddle.static.Executor(place) -- GitLab