build openblas by source (#2154)

* build openblas by source
Signed-off-by: Ncmli <chengming.li@zilliz.com>

* remove openblas md5
Signed-off-by: Ncmli <chengming.li@zilliz.com>
Co-authored-by: Ncmli <chengming.li@zilliz.com>
Co-authored-by: NJin Hai <hai.jin@zilliz.com>

CHANGELOG.md

@@ -10,6 +10,7 @@ Please mark all change in change log and use the issue from GitHub
 -   \#1997 Index file missed after compact
 -   \#2073 Fix CheckDBConfigBackendUrl error message
 -   \#2076 CheckMetricConfigAddress error message
+-   \#1796 Too much low performance of building index on ubuntu-mysql-version
 -   \#2141 Fix server start failed if wal directory exist
 
 ## Feature

core/src/CMakeLists.txt

@@ -320,3 +320,19 @@ install(FILES
         ${CMAKE_BINARY_DIR}/fiu_ep-prefix/src/fiu_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}fiu${CMAKE_SHARED_LIBRARY_SUFFIX}.0
         ${CMAKE_BINARY_DIR}/fiu_ep-prefix/src/fiu_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}fiu${CMAKE_SHARED_LIBRARY_SUFFIX}.1.00
         DESTINATION lib)
+
+if (CMAKE_BUILD_TYPE STREQUAL "Release")
+    install(FILES
+            ${CMAKE_BINARY_DIR}/src/index/openblas_ep-prefix/src/openblas_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}openblas${CMAKE_SHARED_LIBRARY_SUFFIX}
+            ${CMAKE_BINARY_DIR}/src/index/openblas_ep-prefix/src/openblas_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}openblas${CMAKE_SHARED_LIBRARY_SUFFIX}.0
+            ${CMAKE_BINARY_DIR}/src/index/openblas_ep-prefix/src/openblas_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}openblas${CMAKE_SHARED_LIBRARY_SUFFIX}.0.3
+            DESTINATION lib)
+elseif(CMAKE_BUILD_TYPE STREQUAL "Debug")
+    install(FILES
+            ${CMAKE_BINARY_DIR}/src/index/openblas_ep-prefix/src/openblas_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}openblas_d${CMAKE_SHARED_LIBRARY_SUFFIX}
+            ${CMAKE_BINARY_DIR}/src/index/openblas_ep-prefix/src/openblas_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}openblas_d${CMAKE_SHARED_LIBRARY_SUFFIX}.0
+            ${CMAKE_BINARY_DIR}/src/index/openblas_ep-prefix/src/openblas_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}openblas_d${CMAKE_SHARED_LIBRARY_SUFFIX}.0.3
+            DESTINATION lib)
+else()
+    message("unknown CMAKE_BUILD_TYPE")
+endif()
\ No newline at end of file

core/src/index/cmake/DefineOptionsCore.cmake

@@ -73,6 +73,8 @@ define_option(KNOWHERE_BOOST_HEADER_ONLY "Use only BOOST headers" OFF)
 
 define_option(KNOWHERE_WITH_ARROW "Build with ARROW" OFF)
 
+define_option(KNOWHERE_WITH_OPENBLAS "Build with OpenBLAS library" ON)
+
 define_option(KNOWHERE_WITH_FAISS "Build with FAISS library" ON)
 
 define_option(KNOWHERE_WITH_FAISS_GPU_VERSION "Build with FAISS GPU version" ON)

core/src/index/cmake/ThirdPartyPackagesCore.cmake

@@ -14,6 +14,7 @@ set(KNOWHERE_THIRDPARTY_DEPENDENCIES
         Arrow
         FAISS
         GTest
+        OpenBLAS
         MKL
         )
 
@@ -31,6 +32,8 @@ macro(build_dependency DEPENDENCY_NAME)
         build_arrow()
     elseif ("${DEPENDENCY_NAME}" STREQUAL "GTest")
         build_gtest()
+    elseif ("${DEPENDENCY_NAME}" STREQUAL "OpenBLAS")
+        build_openblas()
     elseif ("${DEPENDENCY_NAME}" STREQUAL "FAISS")
         build_faiss()
     elseif ("${DEPENDENCY_NAME}" STREQUAL "MKL")
@@ -217,6 +220,13 @@ else ()
             "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz")
 endif ()
 
+if (DEFINED ENV{KNOWHERE_OPENBLAS_URL})
+    set(OPENBLAS_SOURCE_URL "$ENV{KNOWHERE_OPENBLAS_URL}")
+else ()
+    set(OPENBLAS_SOURCE_URL
+            "https://github.com/xianyi/OpenBLAS/archive/v${OPENBLAS_VERSION}.tar.gz")
+endif ()
+
 # ----------------------------------------------------------------------
 # ARROW
 set(ARROW_PREFIX "${INDEX_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep/cpp")
@@ -303,6 +313,75 @@ if (KNOWHERE_WITH_ARROW AND NOT TARGET arrow_ep)
     include_directories(SYSTEM ${ARROW_INCLUDE_DIR})
 endif ()
 
+# ----------------------------------------------------------------------
+# OpenBLAS
+set(OPENBLAS_PREFIX "${INDEX_BINARY_DIR}/openblas_ep-prefix/src/openblas_ep")
+macro(build_openblas)
+    message(STATUS "Building OpenBLAS-${OPENBLAS_VERSION} from source")
+    set(OPENBLAS_INCLUDE_DIR "${OPENBLAS_PREFIX}/include")
+    if (CMAKE_BUILD_TYPE STREQUAL "Release")
+        set(OPENBLAS_SHARED_LIB
+                "${OPENBLAS_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}openblas${CMAKE_SHARED_LIBRARY_SUFFIX}")
+    elseif(CMAKE_BUILD_TYPE STREQUAL "Debug")
+        set(OPENBLAS_SHARED_LIB
+                "${OPENBLAS_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}openblas_d${CMAKE_SHARED_LIBRARY_SUFFIX}")
+    endif()
+    set(OPENBLAS_STATIC_LIB
+            "${OPENBLAS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}openblas${CMAKE_STATIC_LIBRARY_SUFFIX}")
+    set(OPENBLAS_CMAKE_ARGS
+            ${EP_COMMON_CMAKE_ARGS}
+            -DBUILD_SHARED_LIBS=ON
+            -DBUILD_STATIC_LIBS=ON
+            -DTARGET=CORE2
+            -DDYNAMIC_ARCH=1
+            -DDYNAMIC_OLDER=1
+            -DUSE_THREAD=0
+            -DUSE_OPENMP=0
+            -DFC=gfortran
+            -DCC=gcc
+            -DINTERFACE64=0
+            -DNUM_THREADS=128
+            -DNO_LAPACKE=1
+            "-DVERSION=${VERSION}"
+            "-DCMAKE_INSTALL_PREFIX=${OPENBLAS_PREFIX}"
+            -DCMAKE_INSTALL_LIBDIR=lib)
+
+    externalproject_add(openblas_ep
+            URL
+            ${OPENBLAS_SOURCE_URL}
+            ${EP_LOG_OPTIONS}
+            CMAKE_ARGS
+            ${OPENBLAS_CMAKE_ARGS}
+            BUILD_COMMAND
+            ${MAKE}
+            ${MAKE_BUILD_ARGS}
+            BUILD_IN_SOURCE
+            1
+            INSTALL_COMMAND
+            ${MAKE}
+            PREFIX=${OPENBLAS_PREFIX}
+            install
+            BUILD_BYPRODUCTS
+            ${OPENBLAS_SHARED_LIB}
+            ${OPENBLAS_STATIC_LIB})
+
+    file(MAKE_DIRECTORY "${OPENBLAS_INCLUDE_DIR}")
+    add_library(openblas SHARED IMPORTED)
+    set_target_properties(
+            openblas
+            PROPERTIES IMPORTED_LOCATION "${OPENBLAS_SHARED_LIB}"
+            INTERFACE_INCLUDE_DIRECTORIES "${OPENBLAS_INCLUDE_DIR}")
+
+    add_dependencies(openblas openblas_ep)
+endmacro()
+
+if (KNOWHERE_WITH_OPENBLAS)
+    resolve_dependency(OpenBLAS)
+    get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES)
+    include_directories(SYSTEM "${OPENBLAS_INCLUDE_DIR}")
+    link_directories(SYSTEM ${OPENBLAS_PREFIX}/lib)
+endif()
+
 # ----------------------------------------------------------------------
 # Google gtest
 
@@ -516,7 +595,8 @@ macro(build_faiss)
         set_target_properties(
                 faiss
                 PROPERTIES
-                INTERFACE_LINK_LIBRARIES ${BLAS_LIBRARIES} ${LAPACK_LIBRARIES})
+#                INTERFACE_LINK_LIBRARIES ${BLAS_LIBRARIES} ${LAPACK_LIBRARIES})
+                INTERFACE_LINK_LIBRARIES "openblas")
     endif ()
 
 
@@ -529,15 +609,7 @@ if (KNOWHERE_WITH_FAISS AND NOT TARGET faiss_ep)
     if (FAISS_WITH_MKL)
         resolve_dependency(MKL)
     else ()
-        #    set(BLA_STATIC ON)
-        set(BLA_VENDOR OpenBLAS)
-        find_package(BLAS REQUIRED)
-        #        message(STATUS ${BLAS_LINKER_FLAGS})
-        #        message(STATUS ${BLAS_LIBRARIES})
-        find_package(LAPACK REQUIRED)
-        #        message(STATUS ${LAPACK_LINKER_FLAGS})
-        #        message(STATUS ${LAPACK_LIBRARIES})
-
+        message("faiss with no mkl")
     endif ()
 
     resolve_dependency(FAISS)

core/src/index/thirdparty/versions.txt

@@ -2,5 +2,5 @@ ARROW_VERSION=apache-arrow-0.15.1
 BOOST_VERSION=1.70.0
 GTEST_VERSION=1.8.1
 LAPACK_VERSION=v3.8.0
-OPENBLAS_VERSION=v0.3.6
+OPENBLAS_VERSION=0.3.9
 MKL_VERSION=2019.5.281