diff --git a/ppdet/data/source/coco.py b/ppdet/data/source/coco.py
index 6d8fdcc9e980d5cedc3a90c44f63acb8f5516de3..5c401de14398f72084d8acbd56c1a52325edbe54 100644
--- a/ppdet/data/source/coco.py
+++ b/ppdet/data/source/coco.py
@@ -48,7 +48,7 @@ class COCODataSet(DetDataset):
                  data_fields=['image'],
                  sample_num=-1,
                  load_crowd=False,
-                 allow_empty=True,
+                 allow_empty=False,
                  empty_ratio=1.):
         super(COCODataSet, self).__init__(dataset_dir, image_dir, anno_path,
                                           data_fields, sample_num)
diff --git a/ppdet/data/source/voc.py b/ppdet/data/source/voc.py
index b836b4d052ad3911dfa1ffa4c2de5768e2528c49..1c2a7ef98ccbac760430befc375a79cdebc51a7c 100644
--- a/ppdet/data/source/voc.py
+++ b/ppdet/data/source/voc.py
@@ -55,7 +55,7 @@ class VOCDataSet(DetDataset):
                  data_fields=['image'],
                  sample_num=-1,
                  label_list=None,
-                 allow_empty=True,
+                 allow_empty=False,
                  empty_ratio=1.):
         super(VOCDataSet, self).__init__(
             dataset_dir=dataset_dir,