fix quant_post ce seed (#968)

* fix quant_post ce seed

* fix quant_post ce seed

demo/imagenet_reader.py

@@ -211,13 +211,21 @@ class ImageNetDataset(Dataset):
                 mode='train',
                 color_jitter=False,
                 rotate=False)
-        if self.mode == 'val':
+            return data, np.array([label]).astype('int64')
+        elif self.mode == 'val':
             data, label = process_image(
                 [data_path, sample[1]],
                 mode='val',
                 color_jitter=False,
                 rotate=False)
-        return data, np.array([label]).astype('int64')
+            return data, np.array([label]).astype('int64')
+        elif self.mode == 'test':
+            data = process_image(
+                [data_path, sample[1]],
+                mode='test',
+                color_jitter=False,
+                rotate=False)
+            return data
 
     def __len__(self):
         return len(self.data)

demo/quant/quant_post/eval.py

@@ -31,12 +31,11 @@ add_arg('use_gpu',          bool, True,                 "Whether to use GPU or n
 add_arg('model_path', str,  "./pruning/checkpoints/resnet50/2/eval_model/",                 "Whether to use pretrained model.")
 add_arg('model_name', str,  None, "model filename for inference model")
 add_arg('params_name', str, None, "params filename for inference model")
+add_arg('batch_size',       int,  64,                 "Minibatch size.")
 # yapf: enable
 
 
 def eval(args):
-    # parameters from arguments
-
     place = paddle.CUDAPlace(0) if args.use_gpu else paddle.CPUPlace()
     exe = paddle.static.Executor(place)
 
@@ -45,23 +44,29 @@ def eval(args):
         exe,
         model_filename=args.model_name,
         params_filename=args.params_name)
-    val_reader = paddle.batch(reader.val(), batch_size=1)
+    val_dataset = reader.ImageNetDataset(mode='val')
 
     image = paddle.static.data(
         name='image', shape=[None, 3, 224, 224], dtype='float32')
     label = paddle.static.data(name='label', shape=[None, 1], dtype='int64')
 
-    valid_loader = paddle.io.DataLoader.from_generator(
-        feed_list=[image], capacity=512, use_double_buffer=True, iterable=True)
-    valid_loader.set_sample_list_generator(val_reader, place)
+    val_loader = paddle.io.DataLoader(
+        val_dataset,
+        places=place,
+        feed_list=[image, label],
+        drop_last=False,
+        return_list=True,
+        batch_size=args.batch_size,
+        use_shared_memory=True,
+        shuffle=False)
 
     results = []
-    for batch_id, data in enumerate(val_reader()):
+    for batch_id, data in enumerate(val_loader()):
         # top1_acc, top5_acc
         if len(feed_target_names) == 1:
             # eval "infer model", which input is image, output is classification probability
-            image = data[0][0].reshape((1, 3, 224, 224))
-            label = [[d[1]] for d in data]
+            image = data[0]
+            label = data[1]
             pred = exe.run(val_program,
                            feed={feed_target_names[0]: image},
                            fetch_list=fetch_targets)
@@ -79,8 +84,8 @@ def eval(args):
             results.append([top_1, top_5])
         else:
             # eval "eval model", which inputs are image and label, output is top1 and top5 accuracy
-            image = data[0][0].reshape((1, 3, 224, 224))
-            label = [[d[1]] for d in data]
+            image = data[0]
+            label = data[1]
             result = exe.run(val_program,
                              feed={
                                  feed_target_names[0]: image,
@@ -89,6 +94,8 @@ def eval(args):
                              fetch_list=fetch_targets)
             result = [np.mean(r) for r in result]
             results.append(result)
+        if batch_id % 100 == 0:
+            print('Eval iter: ', batch_id)
     result = np.mean(np.array(results), axis=0)
     print("top1_acc/top5_acc= {}".format(result))
     sys.stdout.flush()

demo/quant/quant_post/quant_post.py

@@ -37,9 +37,28 @@ add_arg('ce_test',                 bool,   False,
 
 
 def quantize(args):
-    val_reader = reader.val()
+    shuffle = True
+    if args.ce_test:
+        # set seed
+        seed = 111
+        np.random.seed(seed)
+        paddle.seed(seed)
+        random.seed(seed)
+        shuffle = False
 
     place = paddle.CUDAPlace(0) if args.use_gpu else paddle.CPUPlace()
+    val_dataset = reader.ImageNetDataset(mode='test')
+    image_shape = [3, 224, 224]
+    image = paddle.static.data(
+        name='image', shape=[None] + image_shape, dtype='float32')
+    data_loader = paddle.io.DataLoader(
+        val_dataset,
+        places=place,
+        feed_list=[image],
+        drop_last=False,
+        return_list=False,
+        batch_size=args.batch_size,
+        shuffle=False)
 
     assert os.path.exists(args.model_path), "args.model_path doesn't exist"
     assert os.path.isdir(args.model_path), "args.model_path must be a dir"
@@ -49,7 +68,7 @@ def quantize(args):
         executor=exe,
         model_dir=args.model_path,
         quantize_model_path=args.save_path,
-        sample_generator=val_reader,
+        data_loader=data_loader,
         model_filename=args.model_filename,
         params_filename=args.params_filename,
         batch_size=args.batch_size,
@@ -62,12 +81,6 @@ def quantize(args):
 def main():
     args = parser.parse_args()
     print_arguments(args)
-    if args.ce_test:
-        # set seed
-        seed = 111
-        np.random.seed(seed)
-        paddle.seed(seed)
-        random.seed(seed)
     quantize(args)