diff --git a/ppstructure/vqa/infer_ser_e2e.py b/ppstructure/vqa/infer_ser_e2e.py
index a90680dafedb6a909dc991edafaa26d7b121797f..33fe4dbb5e809388b135ee467d7e7c230f0eabcc 100644
--- a/ppstructure/vqa/infer_ser_e2e.py
+++ b/ppstructure/vqa/infer_ser_e2e.py
@@ -40,13 +40,6 @@ MODELS = {
     (LayoutLMTokenizer, LayoutLMModel, LayoutLMForTokenClassification)
 }
 
-MODELS = {
-    'LayoutXLM':
-    (LayoutXLMTokenizer, LayoutXLMModel, LayoutXLMForTokenClassification),
-    'LayoutLM':
-    (LayoutLMTokenizer, LayoutLMModel, LayoutLMForTokenClassification)
-}
-
 
 def trans_poly_to_bbox(poly):
     x1 = np.min([p[0] for p in poly])