diff --git a/python/paddle/fluid/__init__.py b/python/paddle/fluid/__init__.py
index 0339abe0960c26f66c3a09178feeaf6550f89777..db6faa1a1b16578b95db4d81ab5bd66e5a003f75 100644
--- a/python/paddle/fluid/__init__.py
+++ b/python/paddle/fluid/__init__.py
@@ -32,10 +32,6 @@ if os.path.exists(legacy_core):
     except Exception as e:
         raise e
 
-# Patch LoDTensor
-from . import core
-core.LoDTensor = core.Tensor
-
 # import all class inside framework into fluid module
 from . import framework
 from .framework import *
diff --git a/python/paddle/fluid/core.py b/python/paddle/fluid/core.py
index 9b99e17e9e51c062477d0832fe07fe966a44cbc6..03b9ad7fc2da411def3819a5a05f8df4ebd3559b 100644
--- a/python/paddle/fluid/core.py
+++ b/python/paddle/fluid/core.py
@@ -253,6 +253,9 @@ load_noavx = False
 
 if avx_supported():
     try:
+        from . import core_avx
+        core_avx.LoDTensor = core_avx.Tensor
+
         from .core_avx import *
         from .core_avx import __doc__, __file__, __name__, __package__
         from .core_avx import __unittest_throw_exception__
@@ -310,6 +313,9 @@ else:
 
 if load_noavx:
     try:
+        from . import core_noavx
+        core_noavx.LoDTensor = core_noavx.Tensor
+
         from .core_noavx import *
         from .core_noavx import __doc__, __file__, __name__, __package__
         from .core_noavx import __unittest_throw_exception__