From f895aebe441b2bef79ecc434626d3cac4b3cbd09 Mon Sep 17 00:00:00 2001 From: Guanghua Yu <742925032@qq.com> Date: Mon, 10 Jan 2022 15:22:13 +0800 Subject: [PATCH] fix quant_post ce seed (#968) * fix quant_post ce seed * fix quant_post ce seed --- demo/imagenet_reader.py | 12 ++++++++++-- demo/quant/quant_post/eval.py | 29 ++++++++++++++++++----------- demo/quant/quant_post/quant_post.py | 29 +++++++++++++++++++++-------- 3 files changed, 49 insertions(+), 21 deletions(-) diff --git a/demo/imagenet_reader.py b/demo/imagenet_reader.py index 050b5c01..28b0fd2c 100644 --- a/demo/imagenet_reader.py +++ b/demo/imagenet_reader.py @@ -211,13 +211,21 @@ class ImageNetDataset(Dataset): mode='train', color_jitter=False, rotate=False) - if self.mode == 'val': + return data, np.array([label]).astype('int64') + elif self.mode == 'val': data, label = process_image( [data_path, sample[1]], mode='val', color_jitter=False, rotate=False) - return data, np.array([label]).astype('int64') + return data, np.array([label]).astype('int64') + elif self.mode == 'test': + data = process_image( + [data_path, sample[1]], + mode='test', + color_jitter=False, + rotate=False) + return data def __len__(self): return len(self.data) diff --git a/demo/quant/quant_post/eval.py b/demo/quant/quant_post/eval.py index a0d8c763..02020424 100755 --- a/demo/quant/quant_post/eval.py +++ b/demo/quant/quant_post/eval.py @@ -31,12 +31,11 @@ add_arg('use_gpu', bool, True, "Whether to use GPU or n add_arg('model_path', str, "./pruning/checkpoints/resnet50/2/eval_model/", "Whether to use pretrained model.") add_arg('model_name', str, None, "model filename for inference model") add_arg('params_name', str, None, "params filename for inference model") +add_arg('batch_size', int, 64, "Minibatch size.") # yapf: enable def eval(args): - # parameters from arguments - place = paddle.CUDAPlace(0) if args.use_gpu else paddle.CPUPlace() exe = paddle.static.Executor(place) @@ -45,23 +44,29 @@ def eval(args): exe, model_filename=args.model_name, params_filename=args.params_name) - val_reader = paddle.batch(reader.val(), batch_size=1) + val_dataset = reader.ImageNetDataset(mode='val') image = paddle.static.data( name='image', shape=[None, 3, 224, 224], dtype='float32') label = paddle.static.data(name='label', shape=[None, 1], dtype='int64') - valid_loader = paddle.io.DataLoader.from_generator( - feed_list=[image], capacity=512, use_double_buffer=True, iterable=True) - valid_loader.set_sample_list_generator(val_reader, place) + val_loader = paddle.io.DataLoader( + val_dataset, + places=place, + feed_list=[image, label], + drop_last=False, + return_list=True, + batch_size=args.batch_size, + use_shared_memory=True, + shuffle=False) results = [] - for batch_id, data in enumerate(val_reader()): + for batch_id, data in enumerate(val_loader()): # top1_acc, top5_acc if len(feed_target_names) == 1: # eval "infer model", which input is image, output is classification probability - image = data[0][0].reshape((1, 3, 224, 224)) - label = [[d[1]] for d in data] + image = data[0] + label = data[1] pred = exe.run(val_program, feed={feed_target_names[0]: image}, fetch_list=fetch_targets) @@ -79,8 +84,8 @@ def eval(args): results.append([top_1, top_5]) else: # eval "eval model", which inputs are image and label, output is top1 and top5 accuracy - image = data[0][0].reshape((1, 3, 224, 224)) - label = [[d[1]] for d in data] + image = data[0] + label = data[1] result = exe.run(val_program, feed={ feed_target_names[0]: image, @@ -89,6 +94,8 @@ def eval(args): fetch_list=fetch_targets) result = [np.mean(r) for r in result] results.append(result) + if batch_id % 100 == 0: + print('Eval iter: ', batch_id) result = np.mean(np.array(results), axis=0) print("top1_acc/top5_acc= {}".format(result)) sys.stdout.flush() diff --git a/demo/quant/quant_post/quant_post.py b/demo/quant/quant_post/quant_post.py index 22f494a5..aeb337fc 100755 --- a/demo/quant/quant_post/quant_post.py +++ b/demo/quant/quant_post/quant_post.py @@ -37,9 +37,28 @@ add_arg('ce_test', bool, False, def quantize(args): - val_reader = reader.val() + shuffle = True + if args.ce_test: + # set seed + seed = 111 + np.random.seed(seed) + paddle.seed(seed) + random.seed(seed) + shuffle = False place = paddle.CUDAPlace(0) if args.use_gpu else paddle.CPUPlace() + val_dataset = reader.ImageNetDataset(mode='test') + image_shape = [3, 224, 224] + image = paddle.static.data( + name='image', shape=[None] + image_shape, dtype='float32') + data_loader = paddle.io.DataLoader( + val_dataset, + places=place, + feed_list=[image], + drop_last=False, + return_list=False, + batch_size=args.batch_size, + shuffle=False) assert os.path.exists(args.model_path), "args.model_path doesn't exist" assert os.path.isdir(args.model_path), "args.model_path must be a dir" @@ -49,7 +68,7 @@ def quantize(args): executor=exe, model_dir=args.model_path, quantize_model_path=args.save_path, - sample_generator=val_reader, + data_loader=data_loader, model_filename=args.model_filename, params_filename=args.params_filename, batch_size=args.batch_size, @@ -62,12 +81,6 @@ def quantize(args): def main(): args = parser.parse_args() print_arguments(args) - if args.ce_test: - # set seed - seed = 111 - np.random.seed(seed) - paddle.seed(seed) - random.seed(seed) quantize(args) -- GitLab