Merge pull request #1988 from FeixLiu/update_dataloader

[bug fixer] Update dataloader to handle index error

.gitignore

@@ -12,3 +12,4 @@ build/
 log/
 nohup.out
 .DS_Store
+.idea

ppcls/static/program.py

@@ -371,6 +371,11 @@ def run(dataloader,
                 "Except RuntimeError when reading data from dataloader, try to read once again..."
             )
             continue
+        except IndexError:
+            logger.warning(
+                "Except IndexError when reading data from dataloader, try to read once again..."
+            )
+            continue
         idx += 1
         # ignore the warmup iters
         if idx == 5:

test_tipc/static/ResNet50/N4C32/ResNet50_bs256_amp_fp16_DP.sh

@@ -8,5 +8,11 @@ num_workers=8
 
 # get data
 bash test_tipc/static/${model_item}/benchmark_common/prepare.sh
+
+cd ./dataset/ILSVRC2012
+cat train_list.txt >> tmp
+for i in {1..10}; do cat tmp >> train_list.txt; done
+cd ../../
+
 # run
 bash test_tipc/static/${model_item}/benchmark_common/run_benchmark.sh ${model_item} ${bs_item} ${fp_item} ${run_mode} ${device_num} ${max_epochs} ${num_workers} 2>&1;

test_tipc/static/ResNet50/N4C32/ResNet50_bs256_fp32_DP.sh

@@ -8,5 +8,11 @@ num_workers=8
 
 # get data
 bash test_tipc/static/${model_item}/benchmark_common/prepare.sh
+
+cd ./dataset/ILSVRC2012
+cat train_list.txt >> tmp
+for i in {1..10}; do cat tmp >> train_list.txt; done
+cd ../../
+
 # run
 bash test_tipc/static/${model_item}/benchmark_common/run_benchmark.sh ${model_item} ${bs_item} ${fp_item} ${run_mode} ${device_num} ${max_epochs} ${num_workers} 2>&1;

test_tipc/static/ResNet50/N4C32/ResNet50_bs256_pure_fp16_DP.sh

@@ -8,5 +8,11 @@ num_workers=8
 
 # get data
 bash test_tipc/static/${model_item}/benchmark_common/prepare.sh
+
+cd ./dataset/ILSVRC2012
+cat train_list.txt >> tmp
+for i in {1..10}; do cat tmp >> train_list.txt; done
+cd ../../
+
 # run
 bash test_tipc/static/${model_item}/benchmark_common/run_benchmark.sh ${model_item} ${bs_item} ${fp_item} ${run_mode} ${device_num} ${max_epochs} ${num_workers} 2>&1;