diff --git a/CMakeLists.txt b/CMakeLists.txt
index e24e71b6998665a1cd82d4eb51cea8dd9dc6772f..22ba9b243dd1522d0c81feef964f824fdd52225b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -228,6 +228,10 @@ if(WITH_MKLML)
     list(APPEND EXTERNAL_LIBS ${MKLML_IOMP_LIB})
 endif()
 
+if(WITH_LIBXSMM)
+    list(APPEND EXTERNAL_LIBS ${LIBXSMM_LIBS})
+endif()
+
 if(WITH_MKLDNN)
     list(APPEND EXTERNAL_LIBS ${MKLDNN_LIB})
 endif()
diff --git a/cmake/external/libxsmm.cmake b/cmake/external/libxsmm.cmake
index d024728080ad188048c571b2ea688244a5354729..f9f1e71311214fad84871bd93dbca916338b6df7 100644
--- a/cmake/external/libxsmm.cmake
+++ b/cmake/external/libxsmm.cmake
@@ -25,11 +25,13 @@ ENDIF()
 
 INCLUDE (ExternalProject)
 
-SET(WITH_SMM ON)
+SET(WITH_LIBXSMM ON)
 SET(LIBXSMM_SOURCES_DIR ${THIRD_PARTY_PATH}/libxsmm)
 SET(LIBXSMM_INSTALL_DIR ${THIRD_PARTY_PATH}/install/libxsmm)
 SET(LIBXSMM_INCLUDE_DIR "${LIBXSMM_INSTALL_DIR}/include" CACHE PATH "LIBXSMM include directory." FORCE)
 SET(LIBXSMM_LIBRARY_DIR "${LIBXSMM_INSTALL_DIR}/lib" CACHE PATH "LIBXSMM library directory." FORCE)
+SET(LIBXSMM_LIBS        "${LIBXSMM_LIBRARY_DIR}/libxsmm.a"
+                        "${LIBXSMM_LIBRARY_DIR}/libxsmmnoblas.a")
 
 ExternalProject_Add(
     extern_libxsmm
@@ -37,14 +39,17 @@ ExternalProject_Add(
     GIT_TAG         "7cc03b5b342fdbc6b6d990b190671c5dbb8489a2"
     PREFIX          ${LIBXSMM_SOURCES_DIR}
     UPDATE_COMMAND  ""
-    BUILD_COMMAND   make PREFIX=${LIBXSMM_INSTALL_DIR} CC=gcc WARP=0 install
+    CONFIGURE_COMMAND ""
+    BUILD_IN_SOURCE 1
+    BUILD_COMMAND   $(MAKE) --silent PREFIX=${LIBXSMM_INSTALL_DIR} CC=gcc WARP=0 install
+    INSTALL_COMMAND ""
 )
-ADD_DEFINITIONS(-DPADDLE_WITH_LIBXSMM)
 ADD_LIBRARY(libxsmm STATIC IMPORTED GLOBAL)
-SET_PROPERTY(TARGET libxsmm PROPERTY IMPORTED_LOCATION
-             "${LIBXSMM_LIBRARY_DIR}/libxsmm.a"
-             "${LIBXSMM_LIBRARY_DIR}/libxsmmnoblas.a")
+SET_PROPERTY(TARGET libxsmm PROPERTY IMPORTED_LOCATION "${LIBXSMM_LIBRARY_DIR}/libxsmm.a")
+SET_PROPERTY(TARGET libxsmm PROPERTY IMPORTED_LOCATION "${LIBXSMM_LIBRARY_DIR}/libxsmmnoblas.a")
 
+MESSAGE(STATUS "Libxsmm library: ${LIBXSMM_LIBS}")
 include_directories(${LIBXSMM_INCLUDE_DIR})
+ADD_DEFINITIONS(-DPADDLE_WITH_LIBXSMM)
 ADD_DEPENDENCIES(libxsmm extern_libxsmm)
 
diff --git a/paddle/fluid/operators/math/CMakeLists.txt b/paddle/fluid/operators/math/CMakeLists.txt
index 5571ff9a7151c1f971ad1805bf001815a651202b..5ae4a00d3f3fb81608d4a4d9dc1162c0efecaabc 100644
--- a/paddle/fluid/operators/math/CMakeLists.txt
+++ b/paddle/fluid/operators/math/CMakeLists.txt
@@ -41,7 +41,7 @@ math_library(depthwise_conv)
 math_library(gru_compute DEPS activation_functions math_function)
 math_library(im2col)
 math_library(lstm_compute DEPS activation_functions)
-cc_library(blas SRCS blas.cc DEPS cblas framework_proto device_context)
+cc_library(blas SRCS blas.cc DEPS cblas libxsmm framework_proto device_context)
 math_library(math_function DEPS blas)
 math_library(maxouting)
 math_library(pooling)
diff --git a/paddle/fluid/operators/math/blas.h b/paddle/fluid/operators/math/blas.h
index 9f6c1e5c35f02cd4bc729eea78b17fac017aa90e..70f88f24f682e05972ca73ef7b50f96be50d1ef4 100644
--- a/paddle/fluid/operators/math/blas.h
+++ b/paddle/fluid/operators/math/blas.h
@@ -21,6 +21,10 @@
 #include "paddle/fluid/platform/dynload/mklml.h"
 #endif
 
+#ifdef PADDLE_WITH_LIBXSMM
+#include <libxsmm.h>
+#endif
+
 #ifdef PADDLE_USE_OPENBLAS
 #include <cblas.h>
 #endif