Skip to content

milvus
milvus

qq_35024513 / milvus
与 Fork 源项目一致

Fork自 milvus / milvus

通知 14
Star 0
Fork 0
milvus
milvus

前往新版Gitcode，体验更适合开发者的 AI 搜索 >>

未验证 提交 a37f9b07 编写于 作者: O op-hunter 提交者： GitHub
浏览文件
操作

Openblas macro (#2269) 

* save local work 4 add openblas macro to chose compile openblas lib or not
Signed-off-by: Ncmli <chengming.li@zilliz.com>

* compile openblas by auto detect
Signed-off-by: Ncmli <chengming.li@zilliz.com>

* Update build environment docker images
Signed-off-by: Nquicksilver <zhifeng.zhang@zilliz.com>

* update changelog
Signed-off-by: Ncmli <chengming.li@zilliz.com>

* Update CMakeLists.txt
Signed-off-by: Nquicksilver <zhifeng.zhang@zilliz.com>

* Update CMakeLists.txt
Signed-off-by: Nquicksilver <zhifeng.zhang@zilliz.com>

* Update CMakeLists.txt
Signed-off-by: Nquicksilver <zhifeng.zhang@zilliz.com>

* Update CMakeLists.txt
Signed-off-by: Nquicksilver <zhifeng.zhang@zilliz.com>

* Update CMakeLists.txt
Signed-off-by: Nquicksilver <zhifeng.zhang@zilliz.com>

* Update CMakeLists.txt
Signed-off-by: Nquicksilver <zhifeng.zhang@zilliz.com>

* Update CMakeLists.txt
Signed-off-by: Nquicksilver <zhifeng.zhang@zilliz.com>

* Update CMakeLists.txt
Signed-off-by: Nquicksilver <zhifeng.zhang@zilliz.com>

* Update CMakeLists.txt
Signed-off-by: Nquicksilver <zhifeng.zhang@zilliz.com>

* Update CMakeLists.txt
Signed-off-by: Nquicksilver <zhifeng.zhang@zilliz.com>

* Update CMakeLists.txt
Signed-off-by: Nquicksilver <zhifeng.zhang@zilliz.com>

* set OpenBLAS_SOURCE auto in build.sh
Signed-off-by: Ncmli <chengming.li@zilliz.com>
Co-authored-by: Ncmli <chengming.li@zilliz.com>
Co-authored-by: Nquicksilver <zhifeng.zhang@zilliz.com>
上级 ca484bbe
隐藏空白更改
内联 并排
Showing with 140 addition and 47 deletion
CHANGELOG.md
浏览文件 @ a37f9b07
......@@ -39,6 +39,7 @@ Please mark all change in change log and use the issue from GitHub
- \#2111 Check GPU environment before start server
- \#2206 Log file rotating
- \#2240 Obtain running rpc requests information
- \#2268 Intelligently detect openblas library in system to avoid installing from source code every time
## Improvement
- \#221 Refactor LOG macro
......
ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml
浏览文件 @ a37f9b07
......@@ -8,7 +8,7 @@ metadata:
spec:
containers:
- name: milvus-cpu-build-env
image: registry.zilliz.com/milvus/milvus-cpu-build-env:v0.7.0-centos7
image: registry.zilliz.com/milvus/milvus-cpu-build-env:v0.9.0-centos7
env:
- name: POD_IP
valueFrom:
......@@ -17,7 +17,7 @@ spec:
- name: OS_NAME
value: "centos7"
- name: BUILD_ENV_IMAGE_ID
value: "225b4d9c26d67b70b476964b4dd6e216de4b464d7a973a8c0c7ed1313c4d81ad"
value: "f2386d84d312e42891c8c70219b12fde014c21fbdbc0e59bede7e7609b1ba58b"
command:
- cat
tty: true
......
ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml
浏览文件 @ a37f9b07
......@@ -8,7 +8,7 @@ metadata:
spec:
containers:
- name: milvus-cpu-build-env
image: registry.zilliz.com/milvus/milvus-cpu-build-env:v0.7.0-ubuntu18.04
image: registry.zilliz.com/milvus/milvus-cpu-build-env:v0.9.0-ubuntu18.04
env:
- name: POD_IP
valueFrom:
......@@ -17,7 +17,7 @@ spec:
- name: OS_NAME
value: "ubuntu18.04"
- name: BUILD_ENV_IMAGE_ID
value: "23476391bec80c64f10d44a6370c73c71f011a6b95114b10ff82a60e771e11c7"
value: "4719a06f1b77393fed7a4336058baab74745715a431193d3876e9b51262505bd"
command:
- cat
tty: true
......
ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml
浏览文件 @ a37f9b07
......@@ -8,7 +8,7 @@ metadata:
spec:
containers:
- name: milvus-gpu-build-env
image: registry.zilliz.com/milvus/milvus-gpu-build-env:v0.7.0-centos7
image: registry.zilliz.com/milvus/milvus-gpu-build-env:v0.9.0-centos7
env:
- name: POD_IP
valueFrom:
......@@ -17,7 +17,7 @@ spec:
- name: OS_NAME
value: "centos7"
- name: BUILD_ENV_IMAGE_ID
value: "a5ec9914737ea4727d88ae36b4a73ca5d817f19438ba913cc1de6a1ee2ed2336"
value: "7087442c4c5a7a7adbd7324c58b7b1ac19a25acfd86d6017b5752c4c6521f90e"
command:
- cat
tty: true
......
ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml
浏览文件 @ a37f9b07
......@@ -8,7 +8,7 @@ metadata:
spec:
containers:
- name: milvus-gpu-build-env
image: registry.zilliz.com/milvus/milvus-gpu-build-env:v0.7.0-ubuntu18.04
image: registry.zilliz.com/milvus/milvus-gpu-build-env:v0.9.0-ubuntu18.04
env:
- name: POD_IP
valueFrom:
......@@ -17,7 +17,7 @@ spec:
- name: OS_NAME
value: "ubuntu18.04"
- name: BUILD_ENV_IMAGE_ID
value: "da9023b0f858f072672f86483a869aa87e90a5140864f89e5a012ec766d96dea"
value: "0aa65ebac377834ceb9644c320f114b97b488d11762948770b994f73e5ae518f"
command:
- cat
tty: true
......
ci/scripts/build.sh
浏览文件 @ a37f9b07
......@@ -123,6 +123,7 @@ CMAKE_CMD="cmake \
-DFAISS_WITH_MKL=${WITH_MKL} \
-DArrow_SOURCE=AUTO \
-DFAISS_SOURCE=AUTO \
-DOpenBLAS_SOURCE=AUTO \
-DMILVUS_WITH_FIU=${FIU_ENABLE} \
${MILVUS_CORE_DIR}"
echo ${CMAKE_CMD}
......
core/build.sh
浏览文件 @ a37f9b07
......@@ -16,6 +16,7 @@ FAISS_ROOT="" #FAISS root path
FAISS_SOURCE="BUNDLED"
WITH_PROMETHEUS="ON"
FIU_ENABLE="OFF"
BUILD_OPENBLAS="ON"
while getopts "p:d:t:f:ulrcghzmei" arg; do
case $arg in
......@@ -112,6 +113,7 @@ CMAKE_CMD="cmake \
-DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
-DFAISS_ROOT=${FAISS_ROOT} \
-DFAISS_SOURCE=${FAISS_SOURCE} \
-DOpenBLAS_SOURCE=AUTO \
-DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \
-DBUILD_COVERAGE=${BUILD_COVERAGE} \
-DMILVUS_DB_PATH=${DB_PATH} \
......@@ -119,7 +121,7 @@ CMAKE_CMD="cmake \
-DMILVUS_GPU_VERSION=${GPU_VERSION} \
-DFAISS_WITH_MKL=${WITH_MKL} \
-DMILVUS_WITH_PROMETHEUS=${WITH_PROMETHEUS} \
-DMILVUS_WITH_FIU=${FIU_ENABLE}
-DMILVUS_WITH_FIU=${FIU_ENABLE} \
../"
echo ${CMAKE_CMD}
${CMAKE_CMD}
......@@ -155,5 +157,5 @@ if [[ ${RUN_CPPLINT} == "ON" ]]; then
else
# compile and build
make -j 8 install || exit 1
make -j 8 VERBOSE=1 install || exit 1
fi
core/src/CMakeLists.txt
浏览文件 @ a37f9b07
......@@ -331,18 +331,10 @@ install(FILES
${CMAKE_BINARY_DIR}/fiu_ep-prefix/src/fiu_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}fiu${CMAKE_SHARED_LIBRARY_SUFFIX}.1.00
DESTINATION lib)
if (CMAKE_BUILD_TYPE STREQUAL "Release")
if(EXISTS ${CMAKE_BINARY_DIR}/src/index/openblas_ep-prefix/src/openblas_ep/lib/)
install(FILES
${CMAKE_BINARY_DIR}/src/index/openblas_ep-prefix/src/openblas_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}openblas${CMAKE_SHARED_LIBRARY_SUFFIX}
${CMAKE_BINARY_DIR}/src/index/openblas_ep-prefix/src/openblas_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}openblas${CMAKE_SHARED_LIBRARY_SUFFIX}.0
${CMAKE_BINARY_DIR}/src/index/openblas_ep-prefix/src/openblas_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}openblas${CMAKE_SHARED_LIBRARY_SUFFIX}.0.3
DESTINATION lib)
elseif(CMAKE_BUILD_TYPE STREQUAL "Debug")
install(FILES
${CMAKE_BINARY_DIR}/src/index/openblas_ep-prefix/src/openblas_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}openblas_d${CMAKE_SHARED_LIBRARY_SUFFIX}
${CMAKE_BINARY_DIR}/src/index/openblas_ep-prefix/src/openblas_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}openblas_d${CMAKE_SHARED_LIBRARY_SUFFIX}.0
${CMAKE_BINARY_DIR}/src/index/openblas_ep-prefix/src/openblas_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}openblas_d${CMAKE_SHARED_LIBRARY_SUFFIX}.0.3
DESTINATION lib)
else()
message("unknown CMAKE_BUILD_TYPE")
endif()
core/src/index/cmake/FindFAISS.cmake
浏览文件 @ a37f9b07
......@@ -38,7 +38,7 @@ if (FAISS_FOUND)
set_target_properties(
faiss
PROPERTIES
INTERFACE_LINK_LIBRARIES ${BLAS_LIBRARIES} ${LAPACK_LIBRARIES})
INTERFACE_LINK_LIBRARIES ${OpenBLAS_LIBRARIES})
endif ()
endif ()
endif ()
core/src/index/cmake/FindOpenBLAS.cmake 0 → 100644
浏览文件 @ a37f9b07
if (OpenBLAS_FOUND) # the git version propose a OpenBLASConfig.cmake
message(STATUS "OpenBLASConfig found")
set(OpenBLAS_INCLUDE_DIR ${OpenBLAS_INCLUDE_DIRS})
else()
message("OpenBLASConfig not found")
unset(OpenBLAS_DIR CACHE)
set(OpenBLAS_INCLUDE_SEARCH_PATHS
/usr/local/openblas/include
/usr/include
/usr/include/openblas
/usr/include/openblas-base
/usr/local/include
/usr/local/include/openblas
/usr/local/include/openblas-base
/opt/OpenBLAS/include
/usr/local/opt/openblas/include
$ENV{OpenBLAS_HOME}
$ENV{OpenBLAS_HOME}/include
)
set(OpenBLAS_LIB_SEARCH_PATHS
/usr/local/openblas/lib
/lib/
/lib/openblas-base
/lib64/
/usr/lib
/usr/lib/openblas-base
/usr/lib64
/usr/local/lib
/usr/local/lib64
/usr/local/opt/openblas/lib
/opt/OpenBLAS/lib
$ENV{OpenBLAS}
$ENV{OpenBLAS}/lib
$ENV{OpenBLAS_HOME}
$ENV{OpenBLAS_HOME}/lib
)
set(DEFAULT_OpenBLAS_LIB_PATH
/usr/local/openblas/lib
${OPENBLAS_PREFIX}/lib)
message("DEFAULT_OpenBLAS_LIB_PATH: ${DEFAULT_OpenBLAS_LIB_PATH}")
find_path(OpenBLAS_INCLUDE_DIR NAMES openblas_config.h lapacke.h PATHS ${OpenBLAS_INCLUDE_SEARCH_PATHS})
find_library(OpenBLAS_LIB NAMES openblas PATHS ${DEFAULT_OpenBLAS_LIB_PATH} NO_DEFAULT_PATH)
find_library(OpenBLAS_LIB NAMES openblas PATHS ${OpenBLAS_LIB_SEARCH_PATHS})
# mostly for debian
find_library(Lapacke_LIB NAMES lapacke PATHS ${DEFAULT_OpenBLAS_LIB_PATH} NO_DEFAULT_PATH)
find_library(Lapacke_LIB NAMES lapacke PATHS ${OpenBLAS_LIB_SEARCH_PATHS})
set(OpenBLAS_FOUND ON)
# Check include files
if(NOT OpenBLAS_INCLUDE_DIR)
set(OpenBLAS_FOUND OFF)
message(STATUS "Could not find OpenBLAS include. Turning OpenBLAS_FOUND off")
else()
message(STATUS "find OpenBLAS include:${OpenBLAS_INCLUDE_DIR} ")
endif()
# Check libraries
if(NOT OpenBLAS_LIB)
set(OpenBLAS_FOUND OFF)
message(STATUS "Could not find OpenBLAS lib. Turning OpenBLAS_FOUND off")
else()
message(STATUS "find OpenBLAS lib:${OpenBLAS_LIB} ")
endif()
if (OpenBLAS_FOUND)
set(OpenBLAS_LIBRARIES ${OpenBLAS_LIB})
STRING(REGEX REPLACE "/libopenblas.so" "" OpenBLAS_LIB_DIR ${OpenBLAS_LIBRARIES})
message(STATUS "find OpenBLAS libraries:${OpenBLAS_LIBRARIES} ")
if (Lapacke_LIB)
set(OpenBLAS_LIBRARIES ${OpenBLAS_LIBRARIES} ${Lapacke_LIB})
endif()
if (NOT OpenBLAS_FIND_QUIETLY)
message(STATUS "Found OpenBLAS libraries: ${OpenBLAS_LIBRARIES}")
message(STATUS "Found OpenBLAS include: ${OpenBLAS_INCLUDE_DIR}")
endif()
else()
if (OpenBLAS_FIND_REQUIRED)
message(FATAL_ERROR "Could not find OpenBLAS")
endif()
endif()
endif()
mark_as_advanced(
OpenBLAS_INCLUDE_DIR
OpenBLAS_LIBRARIES
OpenBLAS_LIB_DIR
)
core/src/index/cmake/ThirdPartyPackagesCore.cmake
浏览文件 @ a37f9b07
......@@ -10,7 +10,6 @@
# or implied. See the License for the specific language governing permissions and limitations under the License.
set(KNOWHERE_THIRDPARTY_DEPENDENCIES
Arrow
FAISS
GTest
......@@ -318,18 +317,15 @@ endif ()
set(OPENBLAS_PREFIX "${INDEX_BINARY_DIR}/openblas_ep-prefix/src/openblas_ep")
macro(build_openblas)
message(STATUS "Building OpenBLAS-${OPENBLAS_VERSION} from source")
set(OPENBLAS_INCLUDE_DIR "${OPENBLAS_PREFIX}/include")
if (CMAKE_BUILD_TYPE STREQUAL "Release")
set(OPENBLAS_SHARED_LIB
"${OPENBLAS_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}openblas${CMAKE_SHARED_LIBRARY_SUFFIX}")
elseif(CMAKE_BUILD_TYPE STREQUAL "Debug")
set(OPENBLAS_SHARED_LIB
"${OPENBLAS_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}openblas_d${CMAKE_SHARED_LIBRARY_SUFFIX}")
endif()
set(OpenBLAS_INCLUDE_DIR "${OPENBLAS_PREFIX}/include")
set(OpenBLAS_LIB_DIR "${OPENBLAS_PREFIX}/lib")
set(OPENBLAS_SHARED_LIB
"${OPENBLAS_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}openblas${CMAKE_SHARED_LIBRARY_SUFFIX}")
set(OPENBLAS_STATIC_LIB
"${OPENBLAS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}openblas${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(OPENBLAS_CMAKE_ARGS
${EP_COMMON_CMAKE_ARGS}
-DCMAKE_BUILD_TYPE=Release
-DBUILD_SHARED_LIBS=ON
-DBUILD_STATIC_LIBS=ON
-DTARGET=CORE2
......@@ -342,7 +338,7 @@ macro(build_openblas)
-DINTERFACE64=0
-DNUM_THREADS=128
-DNO_LAPACKE=1
"-DVERSION=${VERSION}"
"-DVERSION=${OPENBLAS_VERSION}"
"-DCMAKE_INSTALL_PREFIX=${OPENBLAS_PREFIX}"
-DCMAKE_INSTALL_LIBDIR=lib)
......@@ -365,21 +361,23 @@ macro(build_openblas)
${OPENBLAS_SHARED_LIB}
${OPENBLAS_STATIC_LIB})
file(MAKE_DIRECTORY "${OPENBLAS_INCLUDE_DIR}")
file(MAKE_DIRECTORY "${OpenBLAS_INCLUDE_DIR}")
add_library(openblas SHARED IMPORTED)
set_target_properties(
openblas
PROPERTIES IMPORTED_LOCATION "${OPENBLAS_SHARED_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${OPENBLAS_INCLUDE_DIR}")
PROPERTIES
IMPORTED_LOCATION "${OPENBLAS_SHARED_LIB}"
LIBRARY_OUTPUT_NAME "openblas"
INTERFACE_INCLUDE_DIRECTORIES "${OpenBLAS_INCLUDE_DIR}")
add_dependencies(openblas openblas_ep)
get_target_property(OpenBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES)
set(OpenBLAS_LIBRARIES "${OPENBLAS_SHARED_LIB}")
endmacro()
if (KNOWHERE_WITH_OPENBLAS)
resolve_dependency(OpenBLAS)
get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES)
include_directories(SYSTEM "${OPENBLAS_INCLUDE_DIR}")
link_directories(SYSTEM ${OPENBLAS_PREFIX}/lib)
include_directories(SYSTEM "${OpenBLAS_INCLUDE_DIR}")
link_directories(SYSTEM "${OpenBLAS_LIB_DIR}")
endif()
# ----------------------------------------------------------------------
......@@ -525,8 +523,13 @@ macro(build_faiss)
)
else ()
message(STATUS "Build Faiss with OpenBlas/LAPACK")
set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
"LDFLAGS=-L${OPENBLAS_PREFIX}/lib -L${LAPACK_PREFIX}/lib")
if(OpenBLAS_FOUND)
set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
"LDFLAGS=-L${OpenBLAS_LIB_DIR}")
else()
set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
"LDFLAGS=-L${OPENBLAS_PREFIX}/lib")
endif()
endif ()
if (KNOWHERE_GPU_VERSION)
......@@ -577,6 +580,11 @@ macro(build_faiss)
${FAISS_STATIC_LIB})
endif ()
if(NOT OpenBLAS_FOUND)
message("add faiss dependencies: openblas_ep")
ExternalProject_Add_StepDependencies(faiss_ep configure openblas_ep)
endif()
file(MAKE_DIRECTORY "${FAISS_INCLUDE_DIR}")
add_library(faiss STATIC IMPORTED)
......@@ -595,11 +603,9 @@ macro(build_faiss)
set_target_properties(
faiss
PROPERTIES
# INTERFACE_LINK_LIBRARIES ${BLAS_LIBRARIES} ${LAPACK_LIBRARIES})
INTERFACE_LINK_LIBRARIES "openblas")
INTERFACE_LINK_LIBRARIES "${OpenBLAS_LIBRARIES}")
endif ()
add_dependencies(faiss faiss_ep)
endmacro()
......
core/src/index/knowhere/CMakeLists.txt
浏览文件 @ a37f9b07
......@@ -71,7 +71,7 @@ if (FAISS_WITH_MKL)
)
else ()
set(depend_libs ${depend_libs}
${BLAS_LIBRARIES}
${OpenBLAS_LIBRARIES}
${LAPACK_LIBRARIES}
)
endif ()
......@@ -118,7 +118,7 @@ set(INDEX_INCLUDE_DIRS
${INDEX_SOURCE_DIR}/thirdparty/SPTAG/AnnService
# ${ARROW_INCLUDE_DIR}
${FAISS_INCLUDE_DIR}
${OPENBLAS_INCLUDE_DIR}
${OpenBLAS_INCLUDE_DIR}
${LAPACK_INCLUDE_DIR}
)
......
core/src/index/unittest/CMakeLists.txt
浏览文件 @ a37f9b07
......@@ -17,7 +17,7 @@ if (FAISS_WITH_MKL)
)
else ()
set(depend_libs ${depend_libs}
${BLAS_LIBRARIES}
${OpenBLAS_LIBRARIES}
${LAPACK_LIBRARIES}
)
endif ()
......
core/src/index/unittest/faiss_benchmark/CMakeLists.txt
浏览文件 @ a37f9b07
......@@ -24,7 +24,7 @@ if (KNOWHERE_GPU_VERSION)
)
else ()
set(depend_libs ${depend_libs}
${BLAS_LIBRARIES}
${OpenBLAS_LIBRARIES}
${LAPACK_LIBRARIES}
)
endif ()
......
core/src/index/unittest/faiss_ori/CMakeLists.txt
浏览文件 @ a37f9b07
......@@ -19,7 +19,7 @@ if (KNOWHERE_GPU_VERSION)
)
else ()
set(depend_libs ${depend_libs}
${BLAS_LIBRARIES}
${OpenBLAS_LIBRARIES}
${LAPACK_LIBRARIES}
)
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或