diff --git a/core/build.sh b/core/build.sh index f1a9f9406dbb324f0857d244ab7c3a89b3211ed3..5239e7060ceb688a7347284fea3117d5d0e1935a 100755 --- a/core/build.sh +++ b/core/build.sh @@ -8,7 +8,6 @@ MAKE_CLEAN="OFF" BUILD_COVERAGE="OFF" DB_PATH="/tmp/milvus" PROFILING="OFF" -USE_JFROG_CACHE="OFF" RUN_CPPLINT="OFF" CUSTOMIZATION="OFF" # default use ori faiss CUDA_COMPILER=/usr/local/cuda/bin/nvcc @@ -19,7 +18,7 @@ FAISS_SOURCE="BUNDLED" WITH_PROMETHEUS="ON" FIU_ENABLE="OFF" -while getopts "p:d:t:f:ulrcgjhxzmei" arg; do +while getopts "p:d:t:f:ulrcghxzmei" arg; do case $arg in p) INSTALL_PREFIX=$OPTARG @@ -53,9 +52,6 @@ while getopts "p:d:t:f:ulrcgjhxzmei" arg; do z) PROFILING="ON" ;; - j) - USE_JFROG_CACHE="ON" - ;; x) CUSTOMIZATION="ON" ;; @@ -86,7 +82,6 @@ parameter: -r: remove previous build directory(default: OFF) -c: code coverage(default: OFF) -z: profiling(default: OFF) --j: use jfrog cache build directory(default: OFF) -g: build GPU version(default: OFF) -m: build with MKL(default: OFF) -e: build without prometheus(default: OFF) @@ -94,7 +89,7 @@ parameter: -h: help usage: -./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} -f \${FAISS_ROOT} [-u] [-l] [-r] [-c] [-z] [-j] [-g] [-m] [-e] [-h] +./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} -f \${FAISS_ROOT} [-u] [-l] [-r] [-c] [-z] [-g] [-m] [-e] [-h] " exit 0 ;; @@ -125,7 +120,6 @@ CMAKE_CMD="cmake \ -DBUILD_COVERAGE=${BUILD_COVERAGE} \ -DMILVUS_DB_PATH=${DB_PATH} \ -DMILVUS_ENABLE_PROFILING=${PROFILING} \ --DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \ -DCUSTOMIZATION=${CUSTOMIZATION} \ -DMILVUS_GPU_VERSION=${GPU_VERSION} \ -DFAISS_WITH_MKL=${WITH_MKL} \ diff --git a/core/cmake/ThirdPartyPackages.cmake b/core/cmake/ThirdPartyPackages.cmake index 00be339cc222d103ff539b1bd56ff8602855e2de..b6d3b4816aa06c111ad9188280409d48f4d9a2bd 100644 --- a/core/cmake/ThirdPartyPackages.cmake +++ b/core/cmake/ThirdPartyPackages.cmake @@ -131,43 +131,6 @@ endif (UNIX) # thirdparty directory set(THIRDPARTY_DIR "${MILVUS_SOURCE_DIR}/thirdparty") -# ---------------------------------------------------------------------- -# JFrog -if (NOT DEFINED USE_JFROG_CACHE) - set(USE_JFROG_CACHE "OFF") -endif () -if (USE_JFROG_CACHE STREQUAL "ON") - if (DEFINED ENV{JFROG_ARTFACTORY_URL}) - set(JFROG_ARTFACTORY_URL "$ENV{JFROG_ARTFACTORY_URL}") - endif () - if (NOT DEFINED JFROG_ARTFACTORY_URL) - message(FATAL_ERROR "JFROG_ARTFACTORY_URL is not set") - endif () - if (UBUNTU_FOUND) - set(JFROG_ARTFACTORY_CACHE_URL "${JFROG_ARTFACTORY_URL}/milvus/thirdparty/cache/${CMAKE_OS_NAME}/${UBUNTU_VERSION}/${MILVUS_BUILD_ARCH}/${BUILD_TYPE}") - else () - set(JFROG_ARTFACTORY_CACHE_URL "${JFROG_ARTFACTORY_URL}/milvus/thirdparty/cache/${CMAKE_OS_NAME}/${MILVUS_BUILD_ARCH}/${BUILD_TYPE}") - endif () - if (DEFINED ENV{JFROG_USER_NAME}) - set(JFROG_USER_NAME "$ENV{JFROG_USER_NAME}") - endif () - if (NOT DEFINED JFROG_USER_NAME) - message(FATAL_ERROR "JFROG_USER_NAME is not set") - endif () - if (DEFINED ENV{JFROG_PASSWORD}) - set(JFROG_PASSWORD "$ENV{JFROG_PASSWORD}") - endif () - if (NOT DEFINED JFROG_PASSWORD) - message(FATAL_ERROR "JFROG_PASSWORD is not set") - endif () - - set(THIRDPARTY_PACKAGE_CACHE "${THIRDPARTY_DIR}/cache") - if (NOT EXISTS ${THIRDPARTY_PACKAGE_CACHE}) - message(STATUS "Will create cached directory: ${THIRDPARTY_PACKAGE_CACHE}") - file(MAKE_DIRECTORY ${THIRDPARTY_PACKAGE_CACHE}) - endif () -endif () - macro(resolve_dependency DEPENDENCY_NAME) if (${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO") find_package(${DEPENDENCY_NAME} MODULE) @@ -400,48 +363,19 @@ macro(build_gtest) "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gmock${CMAKE_STATIC_LIBRARY_SUFFIX}" ) - if (USE_JFROG_CACHE STREQUAL "ON") - set(GTEST_CACHE_PACKAGE_NAME "googletest_${GTEST_MD5}.tar.gz") - set(GTEST_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${GTEST_CACHE_PACKAGE_NAME}") - set(GTEST_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${GTEST_CACHE_PACKAGE_NAME}") - - file(DOWNLOAD ${GTEST_CACHE_URL} ${GTEST_CACHE_PACKAGE_PATH} STATUS status) - list(GET status 0 status_code) - message(STATUS "DOWNLOADING FROM ${GTEST_CACHE_URL} TO ${GTEST_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") - if (NOT status_code EQUAL 0) - ExternalProject_Add(googletest_ep - URL - ${GTEST_SOURCE_URL} - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - BUILD_BYPRODUCTS - ${GTEST_STATIC_LIB} - ${GTEST_MAIN_STATIC_LIB} - ${GMOCK_STATIC_LIB} - CMAKE_ARGS - ${GTEST_CMAKE_ARGS} - ${EP_LOG_OPTIONS}) - - ExternalProject_Create_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${GTEST_CACHE_URL}) - else () - ExternalProject_Use_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) - endif () - else () - ExternalProject_Add(googletest_ep - URL - ${GTEST_SOURCE_URL} - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - BUILD_BYPRODUCTS - ${GTEST_STATIC_LIB} - ${GTEST_MAIN_STATIC_LIB} - ${GMOCK_STATIC_LIB} - CMAKE_ARGS - ${GTEST_CMAKE_ARGS} - ${EP_LOG_OPTIONS}) - endif () + ExternalProject_Add(googletest_ep + URL + ${GTEST_SOURCE_URL} + BUILD_COMMAND + ${MAKE} + ${MAKE_BUILD_ARGS} + BUILD_BYPRODUCTS + ${GTEST_STATIC_LIB} + ${GTEST_MAIN_STATIC_LIB} + ${GMOCK_STATIC_LIB} + CMAKE_ARGS + ${GTEST_CMAKE_ARGS} + ${EP_LOG_OPTIONS}) # The include directory must exist before it is referenced by a target. file(MAKE_DIRECTORY "${GTEST_INCLUDE_DIR}") @@ -495,52 +429,19 @@ macro(build_mysqlpp) "CXXFLAGS=${EP_CXX_FLAGS}" "LDFLAGS=-pthread") - if (USE_JFROG_CACHE STREQUAL "ON") - set(MYSQLPP_CACHE_PACKAGE_NAME "mysqlpp_${MYSQLPP_MD5}.tar.gz") - set(MYSQLPP_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${MYSQLPP_CACHE_PACKAGE_NAME}") - set(MYSQLPP_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${MYSQLPP_CACHE_PACKAGE_NAME}") - - execute_process(COMMAND wget -q --method HEAD ${MYSQLPP_CACHE_URL} RESULT_VARIABLE return_code) - message(STATUS "Check the remote file ${MYSQLPP_CACHE_URL}. return code = ${return_code}") - if (NOT return_code EQUAL 0) - externalproject_add(mysqlpp_ep - URL - ${MYSQLPP_SOURCE_URL} - ${EP_LOG_OPTIONS} - CONFIGURE_COMMAND - "./configure" - ${MYSQLPP_CONFIGURE_ARGS} - BUILD_COMMAND - ${MAKE} ${MAKE_BUILD_ARGS} - BUILD_IN_SOURCE - 1 - BUILD_BYPRODUCTS - ${MYSQLPP_SHARED_LIB}) - - ExternalProject_Create_Cache(mysqlpp_ep ${MYSQLPP_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/mysqlpp_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${MYSQLPP_CACHE_URL}) - else () - file(DOWNLOAD ${MYSQLPP_CACHE_URL} ${MYSQLPP_CACHE_PACKAGE_PATH} STATUS status) - list(GET status 0 status_code) - message(STATUS "DOWNLOADING FROM ${MYSQLPP_CACHE_URL} TO ${MYSQLPP_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") - if (status_code EQUAL 0) - ExternalProject_Use_Cache(mysqlpp_ep ${MYSQLPP_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) - endif () - endif () - else () - externalproject_add(mysqlpp_ep - URL - ${MYSQLPP_SOURCE_URL} - ${EP_LOG_OPTIONS} - CONFIGURE_COMMAND - "./configure" - ${MYSQLPP_CONFIGURE_ARGS} - BUILD_COMMAND - ${MAKE} ${MAKE_BUILD_ARGS} - BUILD_IN_SOURCE - 1 - BUILD_BYPRODUCTS - ${MYSQLPP_SHARED_LIB}) - endif () + externalproject_add(mysqlpp_ep + URL + ${MYSQLPP_SOURCE_URL} + ${EP_LOG_OPTIONS} + CONFIGURE_COMMAND + "./configure" + ${MYSQLPP_CONFIGURE_ARGS} + BUILD_COMMAND + ${MAKE} ${MAKE_BUILD_ARGS} + BUILD_IN_SOURCE + 1 + BUILD_BYPRODUCTS + ${MYSQLPP_SHARED_LIB}) file(MAKE_DIRECTORY "${MYSQLPP_INCLUDE_DIR}") add_library(mysqlpp SHARED IMPORTED) @@ -586,78 +487,29 @@ macro(build_prometheus) "-DCMAKE_INSTALL_PREFIX=${PROMETHEUS_PREFIX}" -DCMAKE_BUILD_TYPE=Release) - if (USE_JFROG_CACHE STREQUAL "ON") - execute_process(COMMAND sh -c "git ls-remote --heads --tags ${PROMETHEUS_SOURCE_URL} ${PROMETHEUS_VERSION} | cut -f 1" OUTPUT_VARIABLE PROMETHEUS_LAST_COMMIT_ID) - if (${PROMETHEUS_LAST_COMMIT_ID} MATCHES "^[^#][a-z0-9]+") - string(MD5 PROMETHEUS_COMBINE_MD5 "${PROMETHEUS_LAST_COMMIT_ID}") - set(PROMETHEUS_CACHE_PACKAGE_NAME "prometheus_${PROMETHEUS_COMBINE_MD5}.tar.gz") - set(PROMETHEUS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${PROMETHEUS_CACHE_PACKAGE_NAME}") - set(PROMETHEUS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${PROMETHEUS_CACHE_PACKAGE_NAME}") - - execute_process(COMMAND wget -q --method HEAD ${PROMETHEUS_CACHE_URL} RESULT_VARIABLE return_code) - message(STATUS "Check the remote file ${PROMETHEUS_CACHE_URL}. return code = ${return_code}") - if (NOT return_code EQUAL 0) - externalproject_add(prometheus_ep - GIT_REPOSITORY - ${PROMETHEUS_SOURCE_URL} - GIT_TAG - ${PROMETHEUS_VERSION} - GIT_SHALLOW - TRUE - ${EP_LOG_OPTIONS} - CMAKE_ARGS - ${PROMETHEUS_CMAKE_ARGS} - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - BUILD_IN_SOURCE - 1 - INSTALL_COMMAND - ${MAKE} - "DESTDIR=${PROMETHEUS_PREFIX}" - install - BUILD_BYPRODUCTS - "${PROMETHEUS_CORE_STATIC_LIB}" - "${PROMETHEUS_PUSH_STATIC_LIB}" - "${PROMETHEUS_PULL_STATIC_LIB}") - - ExternalProject_Create_Cache(prometheus_ep ${PROMETHEUS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/prometheus_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${PROMETHEUS_CACHE_URL}) - else () - file(DOWNLOAD ${PROMETHEUS_CACHE_URL} ${PROMETHEUS_CACHE_PACKAGE_PATH} STATUS status) - list(GET status 0 status_code) - message(STATUS "DOWNLOADING FROM ${PROMETHEUS_CACHE_URL} TO ${PROMETHEUS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") - if (status_code EQUAL 0) - ExternalProject_Use_Cache(prometheus_ep ${PROMETHEUS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) - endif () - endif () - else () - message(FATAL_ERROR "The last commit ID of \"${PROMETHEUS_SOURCE_URL}\" repository don't match!") - endif () - else () - externalproject_add(prometheus_ep - GIT_REPOSITORY - ${PROMETHEUS_SOURCE_URL} - GIT_TAG - ${PROMETHEUS_VERSION} - GIT_SHALLOW - TRUE - ${EP_LOG_OPTIONS} - CMAKE_ARGS - ${PROMETHEUS_CMAKE_ARGS} - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - BUILD_IN_SOURCE - 1 - INSTALL_COMMAND - ${MAKE} - "DESTDIR=${PROMETHEUS_PREFIX}" - install - BUILD_BYPRODUCTS - "${PROMETHEUS_CORE_STATIC_LIB}" - "${PROMETHEUS_PUSH_STATIC_LIB}" - "${PROMETHEUS_PULL_STATIC_LIB}") - endif () + externalproject_add(prometheus_ep + GIT_REPOSITORY + ${PROMETHEUS_SOURCE_URL} + GIT_TAG + ${PROMETHEUS_VERSION} + GIT_SHALLOW + TRUE + ${EP_LOG_OPTIONS} + CMAKE_ARGS + ${PROMETHEUS_CMAKE_ARGS} + BUILD_COMMAND + ${MAKE} + ${MAKE_BUILD_ARGS} + BUILD_IN_SOURCE + 1 + INSTALL_COMMAND + ${MAKE} + "DESTDIR=${PROMETHEUS_PREFIX}" + install + BUILD_BYPRODUCTS + "${PROMETHEUS_CORE_STATIC_LIB}" + "${PROMETHEUS_PUSH_STATIC_LIB}" + "${PROMETHEUS_PULL_STATIC_LIB}") file(MAKE_DIRECTORY "${PROMETHEUS_PREFIX}/push/include") add_library(prometheus-cpp-push STATIC IMPORTED) @@ -713,54 +565,20 @@ macro(build_sqlite) "CFLAGS=${EP_C_FLAGS}" "CXXFLAGS=${EP_CXX_FLAGS}") - if (USE_JFROG_CACHE STREQUAL "ON") - set(SQLITE_CACHE_PACKAGE_NAME "sqlite_${SQLITE_MD5}.tar.gz") - set(SQLITE_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${SQLITE_CACHE_PACKAGE_NAME}") - set(SQLITE_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${SQLITE_CACHE_PACKAGE_NAME}") - - execute_process(COMMAND wget -q --method HEAD ${SQLITE_CACHE_URL} RESULT_VARIABLE return_code) - message(STATUS "Check the remote file ${SQLITE_CACHE_URL}. return code = ${return_code}") - if (NOT return_code EQUAL 0) - externalproject_add(sqlite_ep - URL - ${SQLITE_SOURCE_URL} - ${EP_LOG_OPTIONS} - CONFIGURE_COMMAND - "./configure" - ${SQLITE_CONFIGURE_ARGS} - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - BUILD_IN_SOURCE - 1 - BUILD_BYPRODUCTS - "${SQLITE_STATIC_LIB}") - - ExternalProject_Create_Cache(sqlite_ep ${SQLITE_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/sqlite_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${SQLITE_CACHE_URL}) - else () - file(DOWNLOAD ${SQLITE_CACHE_URL} ${SQLITE_CACHE_PACKAGE_PATH} STATUS status) - list(GET status 0 status_code) - message(STATUS "DOWNLOADING FROM ${SQLITE_CACHE_URL} TO ${SQLITE_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") - if (status_code EQUAL 0) - ExternalProject_Use_Cache(sqlite_ep ${SQLITE_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) - endif () - endif () - else () - externalproject_add(sqlite_ep - URL - ${SQLITE_SOURCE_URL} - ${EP_LOG_OPTIONS} - CONFIGURE_COMMAND - "./configure" - ${SQLITE_CONFIGURE_ARGS} - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - BUILD_IN_SOURCE - 1 - BUILD_BYPRODUCTS - "${SQLITE_STATIC_LIB}") - endif () + externalproject_add(sqlite_ep + URL + ${SQLITE_SOURCE_URL} + ${EP_LOG_OPTIONS} + CONFIGURE_COMMAND + "./configure" + ${SQLITE_CONFIGURE_ARGS} + BUILD_COMMAND + ${MAKE} + ${MAKE_BUILD_ARGS} + BUILD_IN_SOURCE + 1 + BUILD_BYPRODUCTS + "${SQLITE_STATIC_LIB}") file(MAKE_DIRECTORY "${SQLITE_INCLUDE_DIR}") add_library(sqlite STATIC IMPORTED) @@ -851,48 +669,17 @@ macro(build_yamlcpp) -DYAML_CPP_BUILD_TESTS=OFF -DYAML_CPP_BUILD_TOOLS=OFF) - if (USE_JFROG_CACHE STREQUAL "ON") - set(YAMLCPP_CACHE_PACKAGE_NAME "yaml-cpp_${YAMLCPP_MD5}.tar.gz") - set(YAMLCPP_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${YAMLCPP_CACHE_PACKAGE_NAME}") - set(YAMLCPP_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${YAMLCPP_CACHE_PACKAGE_NAME}") - - execute_process(COMMAND wget -q --method HEAD ${YAMLCPP_CACHE_URL} RESULT_VARIABLE return_code) - message(STATUS "Check the remote file ${YAMLCPP_CACHE_URL}. return code = ${return_code}") - if (NOT return_code EQUAL 0) - externalproject_add(yaml-cpp_ep - URL - ${YAMLCPP_SOURCE_URL} - ${EP_LOG_OPTIONS} - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - BUILD_BYPRODUCTS - "${YAMLCPP_STATIC_LIB}" - CMAKE_ARGS - ${YAMLCPP_CMAKE_ARGS}) - - ExternalProject_Create_Cache(yaml-cpp_ep ${YAMLCPP_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/yaml-cpp_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${YAMLCPP_CACHE_URL}) - else () - file(DOWNLOAD ${YAMLCPP_CACHE_URL} ${YAMLCPP_CACHE_PACKAGE_PATH} STATUS status) - list(GET status 0 status_code) - message(STATUS "DOWNLOADING FROM ${YAMLCPP_CACHE_URL} TO ${YAMLCPP_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") - if (status_code EQUAL 0) - ExternalProject_Use_Cache(yaml-cpp_ep ${YAMLCPP_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) - endif () - endif () - else () - externalproject_add(yaml-cpp_ep - URL - ${YAMLCPP_SOURCE_URL} - ${EP_LOG_OPTIONS} - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - BUILD_BYPRODUCTS - "${YAMLCPP_STATIC_LIB}" - CMAKE_ARGS - ${YAMLCPP_CMAKE_ARGS}) - endif () + externalproject_add(yaml-cpp_ep + URL + ${YAMLCPP_SOURCE_URL} + ${EP_LOG_OPTIONS} + BUILD_COMMAND + ${MAKE} + ${MAKE_BUILD_ARGS} + BUILD_BYPRODUCTS + "${YAMLCPP_STATIC_LIB}" + CMAKE_ARGS + ${YAMLCPP_CMAKE_ARGS}) file(MAKE_DIRECTORY "${YAMLCPP_INCLUDE_DIR}") add_library(yaml-cpp STATIC IMPORTED) @@ -921,56 +708,21 @@ macro(build_libunwind) set(LIBUNWIND_SHARED_LIB "${LIBUNWIND_PREFIX}/lib/libunwind${CMAKE_SHARED_LIBRARY_SUFFIX}") set(LIBUNWIND_CONFIGURE_ARGS "--prefix=${LIBUNWIND_PREFIX}") - if (USE_JFROG_CACHE STREQUAL "ON") - set(LIBUNWIND_CACHE_PACKAGE_NAME "libunwind_${LIBUNWIND_MD5}.tar.gz") - set(LIBUNWIND_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${LIBUNWIND_CACHE_PACKAGE_NAME}") - set(LIBUNWIND_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${LIBUNWIND_CACHE_PACKAGE_NAME}") - - execute_process(COMMAND wget -q --method HEAD ${LIBUNWIND_CACHE_URL} RESULT_VARIABLE return_code) - message(STATUS "Check the remote file ${LIBUNWIND_CACHE_URL}. return code = ${return_code}") - if (NOT return_code EQUAL 0) - externalproject_add(libunwind_ep - URL - ${LIBUNWIND_SOURCE_URL} - ${EP_LOG_OPTIONS} - CONFIGURE_COMMAND - "./configure" - ${LIBUNWIND_CONFIGURE_ARGS} - BUILD_COMMAND - ${MAKE} ${MAKE_BUILD_ARGS} - BUILD_IN_SOURCE - 1 - INSTALL_COMMAND - ${MAKE} install - BUILD_BYPRODUCTS - ${LIBUNWIND_SHARED_LIB}) - - ExternalProject_Create_Cache(libunwind_ep ${LIBUNWIND_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/libunwind_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${LIBUNWIND_CACHE_URL}) - else () - file(DOWNLOAD ${LIBUNWIND_CACHE_URL} ${LIBUNWIND_CACHE_PACKAGE_PATH} STATUS status) - list(GET status 0 status_code) - message(STATUS "DOWNLOADING FROM ${LIBUNWIND_CACHE_URL} TO ${LIBUNWIND_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") - if (status_code EQUAL 0) - ExternalProject_Use_Cache(libunwind_ep ${LIBUNWIND_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) - endif () - endif () - else () - externalproject_add(libunwind_ep - URL - ${LIBUNWIND_SOURCE_URL} - ${EP_LOG_OPTIONS} - CONFIGURE_COMMAND - "./configure" - ${LIBUNWIND_CONFIGURE_ARGS} - BUILD_COMMAND - ${MAKE} ${MAKE_BUILD_ARGS} - BUILD_IN_SOURCE - 1 - INSTALL_COMMAND - ${MAKE} install - BUILD_BYPRODUCTS - ${LIBUNWIND_SHARED_LIB}) - endif () + externalproject_add(libunwind_ep + URL + ${LIBUNWIND_SOURCE_URL} + ${EP_LOG_OPTIONS} + CONFIGURE_COMMAND + "./configure" + ${LIBUNWIND_CONFIGURE_ARGS} + BUILD_COMMAND + ${MAKE} ${MAKE_BUILD_ARGS} + BUILD_IN_SOURCE + 1 + INSTALL_COMMAND + ${MAKE} install + BUILD_BYPRODUCTS + ${LIBUNWIND_SHARED_LIB}) file(MAKE_DIRECTORY "${LIBUNWIND_INCLUDE_DIR}") @@ -999,56 +751,21 @@ macro(build_gperftools) set(GPERFTOOLS_STATIC_LIB "${GPERFTOOLS_PREFIX}/lib/libprofiler${CMAKE_STATIC_LIBRARY_SUFFIX}") set(GPERFTOOLS_CONFIGURE_ARGS "--prefix=${GPERFTOOLS_PREFIX}") - if (USE_JFROG_CACHE STREQUAL "ON") - set(GPERFTOOLS_CACHE_PACKAGE_NAME "gperftools_${GPERFTOOLS_MD5}.tar.gz") - set(GPERFTOOLS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${GPERFTOOLS_CACHE_PACKAGE_NAME}") - set(GPERFTOOLS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${GPERFTOOLS_CACHE_PACKAGE_NAME}") - - execute_process(COMMAND wget -q --method HEAD ${GPERFTOOLS_CACHE_URL} RESULT_VARIABLE return_code) - message(STATUS "Check the remote file ${GPERFTOOLS_CACHE_URL}. return code = ${return_code}") - if (NOT return_code EQUAL 0) - externalproject_add(gperftools_ep - URL - ${GPERFTOOLS_SOURCE_URL} - ${EP_LOG_OPTIONS} - CONFIGURE_COMMAND - "./configure" - ${GPERFTOOLS_CONFIGURE_ARGS} - BUILD_COMMAND - ${MAKE} ${MAKE_BUILD_ARGS} - BUILD_IN_SOURCE - 1 - INSTALL_COMMAND - ${MAKE} install - BUILD_BYPRODUCTS - ${GPERFTOOLS_STATIC_LIB}) - - ExternalProject_Create_Cache(gperftools_ep ${GPERFTOOLS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/gperftools_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${GPERFTOOLS_CACHE_URL}) - else () - file(DOWNLOAD ${GPERFTOOLS_CACHE_URL} ${GPERFTOOLS_CACHE_PACKAGE_PATH} STATUS status) - list(GET status 0 status_code) - message(STATUS "DOWNLOADING FROM ${GPERFTOOLS_CACHE_URL} TO ${GPERFTOOLS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") - if (status_code EQUAL 0) - ExternalProject_Use_Cache(gperftools_ep ${GPERFTOOLS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) - endif () - endif () - else () - externalproject_add(gperftools_ep - URL - ${GPERFTOOLS_SOURCE_URL} - ${EP_LOG_OPTIONS} - CONFIGURE_COMMAND - "./configure" - ${GPERFTOOLS_CONFIGURE_ARGS} - BUILD_COMMAND - ${MAKE} ${MAKE_BUILD_ARGS} - BUILD_IN_SOURCE - 1 - INSTALL_COMMAND - ${MAKE} install - BUILD_BYPRODUCTS - ${GPERFTOOLS_STATIC_LIB}) - endif () + externalproject_add(gperftools_ep + URL + ${GPERFTOOLS_SOURCE_URL} + ${EP_LOG_OPTIONS} + CONFIGURE_COMMAND + "./configure" + ${GPERFTOOLS_CONFIGURE_ARGS} + BUILD_COMMAND + ${MAKE} ${MAKE_BUILD_ARGS} + BUILD_IN_SOURCE + 1 + INSTALL_COMMAND + ${MAKE} install + BUILD_BYPRODUCTS + ${GPERFTOOLS_STATIC_LIB}) ExternalProject_Add_StepDependencies(gperftools_ep build libunwind_ep) @@ -1086,67 +803,26 @@ macro(build_grpc) set(GRPC_PROTOBUF_STATIC_LIB "${GRPC_PROTOBUF_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}protobuf${CMAKE_STATIC_LIBRARY_SUFFIX}") set(GRPC_PROTOC_STATIC_LIB "${GRPC_PROTOBUF_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}protoc${CMAKE_STATIC_LIBRARY_SUFFIX}") - if (USE_JFROG_CACHE STREQUAL "ON") - set(GRPC_CACHE_PACKAGE_NAME "grpc_${GRPC_MD5}.tar.gz") - set(GRPC_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${GRPC_CACHE_PACKAGE_NAME}") - set(GRPC_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${GRPC_CACHE_PACKAGE_NAME}") - - execute_process(COMMAND wget -q --method HEAD ${GRPC_CACHE_URL} RESULT_VARIABLE return_code) - message(STATUS "Check the remote file ${GRPC_CACHE_URL}. return code = ${return_code}") - if (NOT return_code EQUAL 0) - externalproject_add(grpc_ep - URL - ${GRPC_SOURCE_URL} - ${EP_LOG_OPTIONS} - CONFIGURE_COMMAND - "" - BUILD_IN_SOURCE - 1 - BUILD_COMMAND - ${MAKE} ${MAKE_BUILD_ARGS} prefix=${GRPC_PREFIX} - INSTALL_COMMAND - ${MAKE} install prefix=${GRPC_PREFIX} - BUILD_BYPRODUCTS - ${GRPC_STATIC_LIB} - ${GRPC++_STATIC_LIB} - ${GRPCPP_CHANNELZ_STATIC_LIB} - ${GRPC_PROTOBUF_STATIC_LIB} - ${GRPC_PROTOC_STATIC_LIB}) - - ExternalProject_Add_StepDependencies(grpc_ep build zlib_ep) - - ExternalProject_Create_Cache(grpc_ep ${GRPC_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/grpc_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${GRPC_CACHE_URL}) - else () - file(DOWNLOAD ${GRPC_CACHE_URL} ${GRPC_CACHE_PACKAGE_PATH} STATUS status) - list(GET status 0 status_code) - message(STATUS "DOWNLOADING FROM ${GRPC_CACHE_URL} TO ${GRPC_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") - if (status_code EQUAL 0) - ExternalProject_Use_Cache(grpc_ep ${GRPC_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) - endif () - endif () - else () - externalproject_add(grpc_ep - URL - ${GRPC_SOURCE_URL} - ${EP_LOG_OPTIONS} - CONFIGURE_COMMAND - "" - BUILD_IN_SOURCE - 1 - BUILD_COMMAND - ${MAKE} ${MAKE_BUILD_ARGS} prefix=${GRPC_PREFIX} - INSTALL_COMMAND - ${MAKE} install prefix=${GRPC_PREFIX} - BUILD_BYPRODUCTS - ${GRPC_STATIC_LIB} - ${GRPC++_STATIC_LIB} - ${GRPCPP_CHANNELZ_STATIC_LIB} - ${GRPC_PROTOBUF_STATIC_LIB} - ${GRPC_PROTOC_STATIC_LIB}) - - ExternalProject_Add_StepDependencies(grpc_ep build zlib_ep) + externalproject_add(grpc_ep + URL + ${GRPC_SOURCE_URL} + ${EP_LOG_OPTIONS} + CONFIGURE_COMMAND + "" + BUILD_IN_SOURCE + 1 + BUILD_COMMAND + ${MAKE} ${MAKE_BUILD_ARGS} prefix=${GRPC_PREFIX} + INSTALL_COMMAND + ${MAKE} install prefix=${GRPC_PREFIX} + BUILD_BYPRODUCTS + ${GRPC_STATIC_LIB} + ${GRPC++_STATIC_LIB} + ${GRPCPP_CHANNELZ_STATIC_LIB} + ${GRPC_PROTOBUF_STATIC_LIB} + ${GRPC_PROTOC_STATIC_LIB}) - endif () + ExternalProject_Add_StepDependencies(grpc_ep build zlib_ep) file(MAKE_DIRECTORY "${GRPC_INCLUDE_DIR}") @@ -1209,48 +885,17 @@ macro(build_zlib) set(ZLIB_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} "-DCMAKE_INSTALL_PREFIX=${ZLIB_PREFIX}" -DBUILD_SHARED_LIBS=OFF) - if (USE_JFROG_CACHE STREQUAL "ON") - set(ZLIB_CACHE_PACKAGE_NAME "zlib_${ZLIB_MD5}.tar.gz") - set(ZLIB_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${ZLIB_CACHE_PACKAGE_NAME}") - set(ZLIB_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${ZLIB_CACHE_PACKAGE_NAME}") - - execute_process(COMMAND wget -q --method HEAD ${ZLIB_CACHE_URL} RESULT_VARIABLE return_code) - message(STATUS "Check the remote file ${ZLIB_CACHE_URL}. return code = ${return_code}") - if (NOT return_code EQUAL 0) - externalproject_add(zlib_ep - URL - ${ZLIB_SOURCE_URL} - ${EP_LOG_OPTIONS} - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - BUILD_BYPRODUCTS - "${ZLIB_STATIC_LIB}" - CMAKE_ARGS - ${ZLIB_CMAKE_ARGS}) - - ExternalProject_Create_Cache(zlib_ep ${ZLIB_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/zlib_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${ZLIB_CACHE_URL}) - else () - file(DOWNLOAD ${ZLIB_CACHE_URL} ${ZLIB_CACHE_PACKAGE_PATH} STATUS status) - list(GET status 0 status_code) - message(STATUS "DOWNLOADING FROM ${ZLIB_CACHE_URL} TO ${ZLIB_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") - if (status_code EQUAL 0) - ExternalProject_Use_Cache(zlib_ep ${ZLIB_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) - endif () - endif () - else () - externalproject_add(zlib_ep - URL - ${ZLIB_SOURCE_URL} - ${EP_LOG_OPTIONS} - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - BUILD_BYPRODUCTS - "${ZLIB_STATIC_LIB}" - CMAKE_ARGS - ${ZLIB_CMAKE_ARGS}) - endif () + externalproject_add(zlib_ep + URL + ${ZLIB_SOURCE_URL} + ${EP_LOG_OPTIONS} + BUILD_COMMAND + ${MAKE} + ${MAKE_BUILD_ARGS} + BUILD_BYPRODUCTS + "${ZLIB_STATIC_LIB}" + CMAKE_ARGS + ${ZLIB_CMAKE_ARGS}) file(MAKE_DIRECTORY "${ZLIB_INCLUDE_DIR}") add_library(zlib STATIC IMPORTED) diff --git a/core/src/index/build.sh b/core/src/index/build.sh index 357ac5693ac4aa84d645b5f63b7d277707c60348..8557e0e946595bbced7d07b9b4ffe4656a7b19c2 100644 --- a/core/src/index/build.sh +++ b/core/src/index/build.sh @@ -6,9 +6,8 @@ INSTALL_PREFIX=$(pwd)/cmake_build MAKE_CLEAN="OFF" PROFILING="OFF" FAISS_WITH_MKL="OFF" -USE_JFROG_CACHE="OFF" -while getopts "p:d:t:uhrcgmj" arg +while getopts "p:d:t:uhrcgm" arg do case $arg in t) @@ -33,9 +32,6 @@ do m) FAISS_WITH_MKL="ON" ;; - j) - USE_JFROG_CACHE="ON" - ;; h) # help echo " @@ -75,7 +71,6 @@ if [[ ${MAKE_CLEAN} == "ON" ]]; then -DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \ -DMILVUS_ENABLE_PROFILING=${PROFILING} \ -DFAISS_WITH_MKL=${FAISS_WITH_MKL} \ - -DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \ ../" echo ${CMAKE_CMD} diff --git a/core/src/index/cmake/ThirdPartyPackagesCore.cmake b/core/src/index/cmake/ThirdPartyPackagesCore.cmake index ec0df49a48495712e75f1bdcda340a18ed8a979d..b905aed3552e3c0c0ecafe199d1bb133d1762bc2 100644 --- a/core/src/index/cmake/ThirdPartyPackagesCore.cmake +++ b/core/src/index/cmake/ThirdPartyPackagesCore.cmake @@ -115,20 +115,6 @@ endif (UNIX) # thirdparty directory set(THIRDPARTY_DIR "${INDEX_SOURCE_DIR}/thirdparty") -# ---------------------------------------------------------------------- -# JFrog -if (NOT DEFINED USE_JFROG_CACHE) - set(USE_JFROG_CACHE "OFF") -endif () -if (USE_JFROG_CACHE STREQUAL "ON") - set(JFROG_ARTFACTORY_CACHE_URL "${JFROG_ARTFACTORY_URL}/milvus/thirdparty/cache/${CMAKE_OS_NAME}/${KNOWHERE_BUILD_ARCH}/${BUILD_TYPE}") - set(THIRDPARTY_PACKAGE_CACHE "${THIRDPARTY_DIR}/cache") - if (NOT EXISTS ${THIRDPARTY_PACKAGE_CACHE}) - message(STATUS "Will create cached directory: ${THIRDPARTY_PACKAGE_CACHE}") - file(MAKE_DIRECTORY ${THIRDPARTY_PACKAGE_CACHE}) - endif () -endif () - # ---------------------------------------------------------------------- # ExternalProject options @@ -284,71 +270,25 @@ macro(build_arrow) -DBOOST_SOURCE=AUTO #try to find BOOST in the system default locations and build from source if not found ) - - if (USE_JFROG_CACHE STREQUAL "ON") - execute_process(COMMAND sh -c "git ls-remote --heads --tags ${ARROW_SOURCE_URL} ${ARROW_VERSION} | cut -f 1" OUTPUT_VARIABLE ARROW_LAST_COMMIT_ID) - if (${ARROW_LAST_COMMIT_ID} MATCHES "^[^#][a-z0-9]+") - string(MD5 ARROW_COMBINE_MD5 "${ARROW_LAST_COMMIT_ID}") - set(ARROW_CACHE_PACKAGE_NAME "arrow_${ARROW_COMBINE_MD5}.tar.gz") - set(ARROW_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${ARROW_CACHE_PACKAGE_NAME}") - set(ARROW_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${ARROW_CACHE_PACKAGE_NAME}") - - execute_process(COMMAND wget -q --method HEAD ${ARROW_CACHE_URL} RESULT_VARIABLE return_code) - message(STATUS "Check the remote file ${ARROW_CACHE_URL}. return code = ${return_code}") - if (NOT return_code EQUAL 0) - externalproject_add(arrow_ep - GIT_REPOSITORY - ${ARROW_SOURCE_URL} - GIT_TAG - ${ARROW_VERSION} - GIT_SHALLOW - TRUE - SOURCE_SUBDIR - cpp - ${EP_LOG_OPTIONS} - CMAKE_ARGS - ${ARROW_CMAKE_ARGS} - BUILD_COMMAND - "" - INSTALL_COMMAND - ${MAKE} ${MAKE_BUILD_ARGS} install - BUILD_BYPRODUCTS - "${ARROW_STATIC_LIB}" - ) - - ExternalProject_Create_Cache(arrow_ep ${ARROW_CACHE_PACKAGE_PATH} "${INDEX_BINARY_DIR}/arrow_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${ARROW_CACHE_URL}) - else () - file(DOWNLOAD ${ARROW_CACHE_URL} ${ARROW_CACHE_PACKAGE_PATH} STATUS status) - list(GET status 0 status_code) - message(STATUS "DOWNLOADING FROM ${ARROW_CACHE_URL} TO ${ARROW_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") - if (status_code EQUAL 0) - ExternalProject_Use_Cache(arrow_ep ${ARROW_CACHE_PACKAGE_PATH} ${INDEX_BINARY_DIR}) - endif () - endif () - else () - message(FATAL_ERROR "The last commit ID of \"${ARROW_SOURCE_URL}\" repository don't match!") - endif () - else () - externalproject_add(arrow_ep - GIT_REPOSITORY - ${ARROW_SOURCE_URL} - GIT_TAG - ${ARROW_VERSION} - GIT_SHALLOW - TRUE - SOURCE_SUBDIR - cpp - ${EP_LOG_OPTIONS} - CMAKE_ARGS - ${ARROW_CMAKE_ARGS} - BUILD_COMMAND - "" - INSTALL_COMMAND - ${MAKE} ${MAKE_BUILD_ARGS} install - BUILD_BYPRODUCTS - "${ARROW_STATIC_LIB}" - ) - endif () + externalproject_add(arrow_ep + GIT_REPOSITORY + ${ARROW_SOURCE_URL} + GIT_TAG + ${ARROW_VERSION} + GIT_SHALLOW + TRUE + SOURCE_SUBDIR + cpp + ${EP_LOG_OPTIONS} + CMAKE_ARGS + ${ARROW_CMAKE_ARGS} + BUILD_COMMAND + "" + INSTALL_COMMAND + ${MAKE} ${MAKE_BUILD_ARGS} install + BUILD_BYPRODUCTS + "${ARROW_STATIC_LIB}" + ) file(MAKE_DIRECTORY "${ARROW_INCLUDE_DIR}") add_library(arrow STATIC IMPORTED) @@ -409,39 +349,6 @@ macro(build_gtest) "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gmock${CMAKE_STATIC_LIBRARY_SUFFIX}" ) - - if (USE_JFROG_CACHE STREQUAL "ON") - set(GTEST_CACHE_PACKAGE_NAME "googletest_${GTEST_MD5}.tar.gz") - set(GTEST_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${GTEST_CACHE_PACKAGE_NAME}") - set(GTEST_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${GTEST_CACHE_PACKAGE_NAME}") - - execute_process(COMMAND wget -q --method HEAD ${GTEST_CACHE_URL} RESULT_VARIABLE return_code) - message(STATUS "Check the remote file ${GTEST_CACHE_URL}. return code = ${return_code}") - if (NOT return_code EQUAL 0) - ExternalProject_Add(googletest_ep - URL - ${GTEST_SOURCE_URL} - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - BUILD_BYPRODUCTS - ${GTEST_STATIC_LIB} - ${GTEST_MAIN_STATIC_LIB} - ${GMOCK_STATIC_LIB} - CMAKE_ARGS - ${GTEST_CMAKE_ARGS} - ${EP_LOG_OPTIONS}) - - ExternalProject_Create_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} "${INDEX_BINARY_DIR}/googletest_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${GTEST_CACHE_URL}) - else () - file(DOWNLOAD ${GTEST_CACHE_URL} ${GTEST_CACHE_PACKAGE_PATH} STATUS status) - list(GET status 0 status_code) - message(STATUS "DOWNLOADING FROM ${GTEST_CACHE_URL} TO ${GTEST_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") - if (status_code EQUAL 0) - ExternalProject_Use_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} ${INDEX_BINARY_DIR}) - endif () - endif () - else () ExternalProject_Add(googletest_ep URL ${GTEST_SOURCE_URL} @@ -455,7 +362,6 @@ macro(build_gtest) CMAKE_ARGS ${GTEST_CMAKE_ARGS} ${EP_LOG_OPTIONS}) - endif () # The include directory must exist before it is referenced by a target. file(MAKE_DIRECTORY "${GTEST_INCLUDE_DIR}") @@ -564,86 +470,40 @@ macro(build_faiss) set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} --without-cuda) endif () - if (USE_JFROG_CACHE STREQUAL "ON") - string(MD5 FAISS_COMBINE_MD5 "${FAISS_MD5}${LAPACK_MD5}${OPENBLAS_MD5}") - if (KNOWHERE_GPU_VERSION) - set(FAISS_COMPUTE_TYPE "gpu") - else () - set(FAISS_COMPUTE_TYPE "cpu") - endif () - if (FAISS_WITH_MKL) - set(FAISS_CACHE_PACKAGE_NAME "faiss_${FAISS_COMPUTE_TYPE}_mkl_${FAISS_COMBINE_MD5}.tar.gz") - else () - set(FAISS_CACHE_PACKAGE_NAME "faiss_${FAISS_COMPUTE_TYPE}_openblas_${FAISS_COMBINE_MD5}.tar.gz") - endif () - set(FAISS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${FAISS_CACHE_PACKAGE_NAME}") - set(FAISS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${FAISS_CACHE_PACKAGE_NAME}") - - execute_process(COMMAND wget -q --method HEAD ${FAISS_CACHE_URL} RESULT_VARIABLE return_code) - message(STATUS "Check the remote file ${FAISS_CACHE_URL}. return code = ${return_code}") - if (NOT return_code EQUAL 0) - externalproject_add(faiss_ep - URL - ${FAISS_SOURCE_URL} - ${EP_LOG_OPTIONS} - CONFIGURE_COMMAND - "./configure" - ${FAISS_CONFIGURE_ARGS} - BUILD_COMMAND - ${MAKE} ${MAKE_BUILD_ARGS} all - BUILD_IN_SOURCE - 1 - INSTALL_COMMAND - ${MAKE} install - BUILD_BYPRODUCTS - ${FAISS_STATIC_LIB}) - - ExternalProject_Create_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} "${INDEX_BINARY_DIR}/faiss_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${FAISS_CACHE_URL}) - else () - file(DOWNLOAD ${FAISS_CACHE_URL} ${FAISS_CACHE_PACKAGE_PATH} STATUS status) - list(GET status 0 status_code) - message(STATUS "DOWNLOADING FROM ${FAISS_CACHE_URL} TO ${FAISS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") - if (status_code EQUAL 0) - ExternalProject_Use_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} ${INDEX_BINARY_DIR}) - endif () - endif () + if (CUSTOMIZATION) + externalproject_add(faiss_ep + DOWNLOAD_COMMAND + "" + SOURCE_DIR + ${FAISS_SOURCE_DIR} + ${EP_LOG_OPTIONS} + CONFIGURE_COMMAND + "./configure" + ${FAISS_CONFIGURE_ARGS} + BUILD_COMMAND + ${MAKE} ${MAKE_BUILD_ARGS} all + BUILD_IN_SOURCE + 1 + INSTALL_COMMAND + ${MAKE} install + BUILD_BYPRODUCTS + ${FAISS_STATIC_LIB}) else () - if (CUSTOMIZATION) - externalproject_add(faiss_ep - DOWNLOAD_COMMAND - "" - SOURCE_DIR - ${FAISS_SOURCE_DIR} - ${EP_LOG_OPTIONS} - CONFIGURE_COMMAND - "./configure" - ${FAISS_CONFIGURE_ARGS} - BUILD_COMMAND - ${MAKE} ${MAKE_BUILD_ARGS} all - BUILD_IN_SOURCE - 1 - INSTALL_COMMAND - ${MAKE} install - BUILD_BYPRODUCTS - ${FAISS_STATIC_LIB}) - else () - externalproject_add(faiss_ep - URL - ${FAISS_SOURCE_URL} - ${EP_LOG_OPTIONS} - CONFIGURE_COMMAND - "./configure" - ${FAISS_CONFIGURE_ARGS} - BUILD_COMMAND - ${MAKE} ${MAKE_BUILD_ARGS} all - BUILD_IN_SOURCE - 1 - INSTALL_COMMAND - ${MAKE} install - BUILD_BYPRODUCTS - ${FAISS_STATIC_LIB}) - endif () - + externalproject_add(faiss_ep + URL + ${FAISS_SOURCE_URL} + ${EP_LOG_OPTIONS} + CONFIGURE_COMMAND + "./configure" + ${FAISS_CONFIGURE_ARGS} + BUILD_COMMAND + ${MAKE} ${MAKE_BUILD_ARGS} all + BUILD_IN_SOURCE + 1 + INSTALL_COMMAND + ${MAKE} install + BUILD_BYPRODUCTS + ${FAISS_STATIC_LIB}) endif () file(MAKE_DIRECTORY "${FAISS_INCLUDE_DIR}") diff --git a/core/src/index/thirdparty/faiss/ci/jenkins/Jenkinsfile b/core/src/index/thirdparty/faiss/ci/jenkins/Jenkinsfile deleted file mode 100644 index 347e53a69369ce97e0eff20329eae2315cfaa3d8..0000000000000000000000000000000000000000 --- a/core/src/index/thirdparty/faiss/ci/jenkins/Jenkinsfile +++ /dev/null @@ -1,579 +0,0 @@ -pipeline { - agent none - - options { - timestamps() - } - - parameters{ - choice choices: ['Release', 'Debug'], description: 'Build Type', name: 'BUILD_TYPE' - choice choices: ['False', 'True'], description: 'Whether it is origin Faiss', name: 'IS_ORIGIN_FAISS' - string defaultValue: 'registry.zilliz.com', description: 'DOCKER REGISTRY URL', name: 'DOKCER_REGISTRY_URL', trim: true - string defaultValue: 'a54e38ef-c424-4ea9-9224-b25fc20e3924', description: 'DOCKER CREDENTIALS ID', name: 'DOCKER_CREDENTIALS_ID', trim: true - string defaultValue: 'http://192.168.1.201/artifactory/milvus', description: 'JFROG ARTFACTORY URL', name: 'JFROG_ARTFACTORY_URL', trim: true - string defaultValue: '76fd48ab-2b8e-4eed-834d-2eefd23bb3a6', description: 'JFROG CREDENTIALS ID', name: 'JFROG_CREDENTIALS_ID', trim: true - } - - environment { - PROJECT_NAME = "milvus" - LOWER_BUILD_TYPE = params.BUILD_TYPE.toLowerCase() - SEMVER = "0.6.0" - JOBNAMES = env.JOB_NAME.split('/') - PIPELINE_NAME = "${JOBNAMES[0]}" - FAISS_ROOT_PATH="/usr/local/faiss" - NATIVE_FAISS_VERSION="1.6.0" - } - - stages { - stage("Ubuntu 18.04 x86_64") { - environment { - OS_NAME = "ubuntu18.04" - CPU_ARCH = "amd64" - } - - parallel { - stage("GPU Version") { - environment { - BINRARY_VERSION = "gpu" - PACKAGE_VERSION = VersionNumber([ - versionNumberString : '${SEMVER}-gpu-${OS_NAME}-${CPU_ARCH}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}' - ]); - DOCKER_VERSION = "${SEMVER}-gpu-${OS_NAME}-${LOWER_BUILD_TYPE}" - } - - stages { - stage("Run Build") { - agent { - kubernetes { - label "${env.BINRARY_VERSION}-build" - defaultContainer 'jnlp' - yaml """ -apiVersion: v1 -kind: Pod -metadata: - name: milvus-gpu-build-env - labels: - app: milvus - componet: gpu-build-env -spec: - containers: - - name: milvus-gpu-build-env - image: registry.zilliz.com/milvus/milvus-gpu-build-env:v0.6.0-ubuntu18.04 - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: BUILD_ENV_IMAGE_ID - value: "da9023b0f858f072672f86483a869aa87e90a5140864f89e5a012ec766d96dea" - command: - - cat - tty: true - resources: - limits: - memory: "24Gi" - cpu: "8.0" - nvidia.com/gpu: 1 - requests: - memory: "16Gi" - cpu: "4.0" - - name: milvus-mysql - image: mysql:5.6 - env: - - name: MYSQL_ROOT_PASSWORD - value: 123456 - ports: - - containerPort: 3306 - name: mysql - """ - } - } - - stages { - stage('Prepare') { - steps { - container("milvus-${env.BINRARY_VERSION}-build-env") { - script { - dir ("milvus") { - checkout([$class: 'GitSCM', branches: [[name: "${env.SEMVER}"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus.git", name: 'origin', refspec: "+refs/heads/${env.SEMVER}:refs/remotes/origin/${env.SEMVER}"]]]) - } - } - } - } - } - - stage('Build') { - steps { - container("milvus-${env.BINRARY_VERSION}-build-env") { - script { - load "${env.WORKSPACE}/ci/jenkins/step/build.groovy" - } - } - } - } - stage('Code Coverage') { - steps { - container("milvus-${env.BINRARY_VERSION}-build-env") { - script { - load "${env.WORKSPACE}/ci/jenkins/step/coverage.groovy" - } - } - } - } - stage('Upload Package') { - steps { - container("milvus-${env.BINRARY_VERSION}-build-env") { - script { - dir ("milvus") { - load "ci/jenkins/step/package.groovy" - } - } - } - } - } - } - } - - stage("Publish docker images") { - agent { - kubernetes { - label "${BINRARY_VERSION}-publish" - defaultContainer 'jnlp' - yaml """ -apiVersion: v1 -kind: Pod -metadata: - labels: - app: publish - componet: docker -spec: - containers: - - name: publish-images - image: registry.zilliz.com/library/docker:v1.0.0 - securityContext: - privileged: true - command: - - cat - tty: true - volumeMounts: - - name: docker-sock - mountPath: /var/run/docker.sock - volumes: - - name: docker-sock - hostPath: - path: /var/run/docker.sock -""" - } - } - - stages { - stage('Prepare') { - steps { - container('publish-images') { - script { - dir ("milvus") { - checkout([$class: 'GitSCM', branches: [[name: "${env.SEMVER}"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus.git", name: 'origin', refspec: "+refs/heads/${env.SEMVER}:refs/remotes/origin/${env.SEMVER}"]]]) - } - } - } - } - } - - stage('Publish') { - steps { - container('publish-images') { - script { - dir ("milvus") { - load "ci/jenkins/step/publishImages.groovy" - } - } - } - } - } - } - } - - stage("Deploy to Development") { - environment { - FROMAT_SEMVER = "${env.SEMVER}".replaceAll("\\.", "-") - HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.FROMAT_SEMVER}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}".toLowerCase() - } - - agent { - kubernetes { - label "${env.BINRARY_VERSION}-dev-test" - defaultContainer 'jnlp' - yaml """ -apiVersion: v1 -kind: Pod -metadata: - labels: - app: milvus - componet: test-env -spec: - containers: - - name: milvus-test-env - image: registry.zilliz.com/milvus/milvus-test-env:v0.1 - command: - - cat - tty: true - volumeMounts: - - name: kubeconf - mountPath: /root/.kube/ - readOnly: true - volumes: - - name: kubeconf - secret: - secretName: test-cluster-config -""" - } - } - - stages { - stage('Prepare') { - steps { - container('milvus-test-env') { - script { - dir ("milvus") { - checkout([$class: 'GitSCM', branches: [[name: "${env.SEMVER}"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus.git", name: 'origin', refspec: "+refs/heads/${env.SEMVER}:refs/remotes/origin/${env.SEMVER}"]]]) - } - } - } - } - } - - stage("Deploy to Dev") { - steps { - container('milvus-test-env') { - script { - dir ("milvus") { - load "ci/jenkins/step/deploySingle2Dev.groovy" - } - } - } - } - } - - stage("Dev Test") { - steps { - container('milvus-test-env') { - script { - boolean isNightlyTest = isTimeTriggeredBuild() - if (isNightlyTest) { - dir ("milvus") { - load "ci/jenkins/step/singleDevNightlyTest.groovy" - } - } else { - dir ("milvus") { - load "ci/jenkins/step/singleDevTest.groovy" - } - } - } - } - } - } - - stage ("Cleanup Dev") { - steps { - container('milvus-test-env') { - script { - dir ("milvus") { - load "ci/jenkins/step/cleanupSingleDev.groovy" - } - } - } - } - } - } - post { - unsuccessful { - container('milvus-test-env') { - script { - dir ("milvus") { - load "ci/jenkins/step/cleanupSingleDev.groovy" - } - } - } - } - } - } - } - } - - stage("CPU Version") { - environment { - BINRARY_VERSION = "cpu" - PACKAGE_VERSION = VersionNumber([ - versionNumberString : '${SEMVER}-cpu-${OS_NAME}-${CPU_ARCH}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}' - ]); - DOCKER_VERSION = "${SEMVER}-cpu-${OS_NAME}-${LOWER_BUILD_TYPE}" - } - - stages { - stage("Run Build") { - agent { - kubernetes { - label "${env.BINRARY_VERSION}-build" - defaultContainer 'jnlp' - yaml """ -apiVersion: v1 -kind: Pod -metadata: - name: milvus-cpu-build-env - labels: - app: milvus - componet: cpu-build-env -spec: - containers: - - name: milvus-cpu-build-env - image: registry.zilliz.com/milvus/milvus-cpu-build-env:v0.6.0-ubuntu18.04 - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: BUILD_ENV_IMAGE_ID - value: "23476391bec80c64f10d44a6370c73c71f011a6b95114b10ff82a60e771e11c7" - command: - - cat - tty: true - resources: - limits: - memory: "24Gi" - cpu: "8.0" - requests: - memory: "16Gi" - cpu: "4.0" - - name: milvus-mysql - image: mysql:5.6 - env: - - name: MYSQL_ROOT_PASSWORD - value: 123456 - ports: - - containerPort: 3306 - name: mysql - """ - } - } - - stages { - stage('Prepare') { - steps { - container("milvus-${env.BINRARY_VERSION}-build-env") { - script { - dir ("milvus") { - checkout([$class: 'GitSCM', branches: [[name: "${env.SEMVER}"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus.git", name: 'origin', refspec: "+refs/heads/${env.SEMVER}:refs/remotes/origin/${env.SEMVER}"]]]) - } - } - } - } - } - - stage('Build') { - steps { - container("milvus-${env.BINRARY_VERSION}-build-env") { - script { - load "${env.WORKSPACE}/ci/jenkins/step/build.groovy" - } - } - } - } - stage('Code Coverage') { - steps { - container("milvus-${env.BINRARY_VERSION}-build-env") { - script { - load "${env.WORKSPACE}/ci/jenkins/step/coverage.groovy" - } - } - } - } - stage('Upload Package') { - steps { - container("milvus-${env.BINRARY_VERSION}-build-env") { - script { - dir ("milvus") { - load "ci/jenkins/step/package.groovy" - } - } - } - } - } - } - } - - stage("Publish docker images") { - agent { - kubernetes { - label "${BINRARY_VERSION}-publish" - defaultContainer 'jnlp' - yaml """ -apiVersion: v1 -kind: Pod -metadata: - labels: - app: publish - componet: docker -spec: - containers: - - name: publish-images - image: registry.zilliz.com/library/docker:v1.0.0 - securityContext: - privileged: true - command: - - cat - tty: true - volumeMounts: - - name: docker-sock - mountPath: /var/run/docker.sock - volumes: - - name: docker-sock - hostPath: - path: /var/run/docker.sock -""" - } - } - - stages { - stage('Prepare') { - steps { - container("publish-images") { - script { - dir ("milvus") { - checkout([$class: 'GitSCM', branches: [[name: "${env.SEMVER}"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus.git", name: 'origin', refspec: "+refs/heads/${env.SEMVER}:refs/remotes/origin/${env.SEMVER}"]]]) - } - } - } - } - } - - stage('Publish') { - steps { - container('publish-images'){ - script { - dir ("milvus") { - load "ci/jenkins/step/publishImages.groovy" - } - } - } - } - } - } - } - - stage("Deploy to Development") { - environment { - FROMAT_SEMVER = "${env.SEMVER}".replaceAll("\\.", "-") - HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.FROMAT_SEMVER}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}".toLowerCase() - } - - agent { - kubernetes { - label "${env.BINRARY_VERSION}-dev-test" - defaultContainer 'jnlp' - yaml """ -apiVersion: v1 -kind: Pod -metadata: - labels: - app: milvus - componet: test-env -spec: - containers: - - name: milvus-test-env - image: registry.zilliz.com/milvus/milvus-test-env:v0.1 - command: - - cat - tty: true - volumeMounts: - - name: kubeconf - mountPath: /root/.kube/ - readOnly: true - volumes: - - name: kubeconf - secret: - secretName: test-cluster-config -""" - } - } - - stages { - stage('Prepare') { - steps { - container('milvus-test-env') { - script { - dir ("milvus") { - checkout([$class: 'GitSCM', branches: [[name: "${env.SEMVER}"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus.git", name: 'origin', refspec: "+refs/heads/${env.SEMVER}:refs/remotes/origin/${env.SEMVER}"]]]) - } - } - } - } - } - - stage("Deploy to Dev") { - steps { - container('milvus-test-env') { - script { - dir ("milvus") { - load "ci/jenkins/step/deploySingle2Dev.groovy" - } - } - } - } - } - - stage("Dev Test") { - steps { - container('milvus-test-env') { - script { - boolean isNightlyTest = isTimeTriggeredBuild() - if (isNightlyTest) { - dir ("milvus") { - load "ci/jenkins/step/singleDevNightlyTest.groovy" - } - } else { - dir ("milvus") { - load "ci/jenkins/step/singleDevTest.groovy" - } - } - } - } - } - } - - stage ("Cleanup Dev") { - steps { - container('milvus-test-env') { - script { - dir ("milvus") { - load "ci/jenkins/step/cleanupSingleDev.groovy" - } - } - } - } - } - } - post { - unsuccessful { - container('milvus-test-env') { - script { - dir ("milvus") { - load "ci/jenkins/step/cleanupSingleDev.groovy" - } - } - } - } - } - } - } - } - } - } - } -} - -boolean isTimeTriggeredBuild() { - if (currentBuild.getBuildCauses('hudson.triggers.TimerTrigger$TimerTriggerCause').size() != 0) { - return true - } - return false -} diff --git a/core/src/index/thirdparty/faiss/ci/jenkins/step/build.groovy b/core/src/index/thirdparty/faiss/ci/jenkins/step/build.groovy deleted file mode 100644 index 6013aea5ea067351f964af4c2e8eb33fdb9f270a..0000000000000000000000000000000000000000 --- a/core/src/index/thirdparty/faiss/ci/jenkins/step/build.groovy +++ /dev/null @@ -1,35 +0,0 @@ -timeout(time: 60, unit: 'MINUTES') { - dir ("ci/scripts") { - if ("${env.BINRARY_VERSION}" == "gpu") { - if ("${params.IS_ORIGIN_FAISS}" == "False") { - sh "./build.sh -o ${env.FAISS_ROOT_PATH} -i -g" - } else { - sh "wget https://github.com/facebookresearch/faiss/archive/v${env.NATIVE_FAISS_VERSION}.tar.gz && \ - tar zxvf v${env.NATIVE_FAISS_VERSION}.tar.gz" - sh "./build.sh -o ${env.FAISS_ROOT_PATH} -s ./faiss-${env.NATIVE_FAISS_VERSION} -i -g" - } - } else { - sh "wget https://github.com/facebookresearch/faiss/archive/v${env.NATIVE_FAISS_VERSION}.tar.gz && \ - tar zxvf v${env.NATIVE_FAISS_VERSION}.tar.gz" - sh "./build.sh -o ${env.FAISS_ROOT_PATH} -s ./faiss-${env.NATIVE_FAISS_VERSION} -i" - } - } - - dir ("milvus") { - dir ("ci/scripts") { - withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { - def checkResult = sh(script: "./check_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache", returnStatus: true) - if ("${env.BINRARY_VERSION}" == "gpu") { - if ("${params.IS_ORIGIN_FAISS}" == "False") { - sh ". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -f ${env.FAISS_ROOT_PATH} -l -m -g -x -u -c" - } else { - sh ". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -f ${env.FAISS_ROOT_PATH} -l -m -g -u -c" - } - } else { - sh ". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -f ${env.FAISS_ROOT_PATH} -l -m -u -c" - } - sh "./update_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache -u ${USERNAME} -p ${PASSWORD}" - } - } - } -} diff --git a/core/src/index/thirdparty/faiss/ci/jenkins/step/coverage.groovy b/core/src/index/thirdparty/faiss/ci/jenkins/step/coverage.groovy deleted file mode 100644 index 40d2d2f03a03d9274b0127295f209d60d3796c00..0000000000000000000000000000000000000000 --- a/core/src/index/thirdparty/faiss/ci/jenkins/step/coverage.groovy +++ /dev/null @@ -1,5 +0,0 @@ -timeout(time: 30, unit: 'MINUTES') { - dir ("milvus/ci/scripts") { - sh "./coverage.sh -o /opt/milvus -u root -p 123456 -t \$POD_IP" - } -} \ No newline at end of file diff --git a/core/src/index/thirdparty/faiss/ci/scripts/build.sh b/core/src/index/thirdparty/faiss/ci/scripts/build.sh deleted file mode 100755 index f3bb2dcaca66ce15bf2fc987dc53019058640da0..0000000000000000000000000000000000000000 --- a/core/src/index/thirdparty/faiss/ci/scripts/build.sh +++ /dev/null @@ -1,127 +0,0 @@ -#!/bin/bash - -set -e - -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink - DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -done -SCRIPTS_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" - -FAISS_SOURCE_DIR="${SCRIPTS_DIR}/../.." - -FAISS_WITH_MKL="False" -FAISS_GPU_VERSION="False" -FAISS_COMMON_CONFIGURE_ARGS="CXXFLAGS=\"-mavx2 -mf16c\" --without-python" -FAISS_CONFIGURE_ARGS="${FAISS_COMMON_CONFIGURE_ARGS}" -CUDA_TOOLKIT_ROOT_DIR="/usr/local/cuda" -FAISS_CUDA_ARCH="-gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_61,code=sm_61 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75" -MKL_ROOT_DIR="/opt/intel/compilers_and_libraries_2019.5.281/linux/mkl" - -while getopts "o:s:m:b:l:c:a:igh" arg -do - case $arg in - o) - FAISS_INSTALL_PREFIX=$OPTARG - ;; - s) - FAISS_SOURCE_DIR=$OPTARG - ;; - m) - MKL_ROOT_DIR=$OPTARG - ;; - b) - OPENBLAS_PREFIX=$OPTARG - ;; - l) - LAPACK_PREFIX=$OPTARG - ;; - c) - CUDA_TOOLKIT_ROOT_DIR=$OPTARG - ;; - a) - FAISS_CUDA_ARCH=$OPTARG - ;; - i) - FAISS_WITH_MKL="True" - ;; - g) - FAISS_GPU_VERSION="True" - ;; - h) # help - echo " - -parameter: --o: faiss install prefix path --s: faiss source directory --m: mkl root directory --b: openblas install prefix path --l: lapack install prefix path --c: CUDA toolkit root directory --a: faiss CUDA compute architecture --i: faiss with mkl --g: faiss gpu version --h: help - -usage: -./build.sh -o \${FAISS_INSTALL_PREFIX} -s \${FAISS_SOURCE_DIR} -m \${MKL_ROOT_DIR} -b \${OPENBLAS_PREFIX} -l \${LAPACK_PREFIX} -c \${CUDA_TOOLKIT_ROOT_DIR} -a \${FAISS_CUDA_ARCH} [-i] [-g] [-h] - " - exit 0 - ;; - ?) - echo "ERROR! unknown argument" - exit 1 - ;; - esac -done - -if [[ -n "${FAISS_INSTALL_PREFIX}" ]];then - FAISS_CONFIGURE_ARGS="${FAISS_CONFIGURE_ARGS} --prefix=${FAISS_INSTALL_PREFIX}" -fi - -if [[ "${FAISS_GPU_VERSION}" == "True" ]];then - if [[ ! -n "${FAISS_CUDA_ARCH}" ]];then - echo "FAISS_CUDA_ARCH: \"${FAISS_CUDA_ARCH}\" is empty!" - exit 1 - fi - if [[ ! -d "${CUDA_TOOLKIT_ROOT_DIR}" ]];then - echo "CUDA_TOOLKIT_ROOT_DIR: \"${CUDA_TOOLKIT_ROOT_DIR}\" directory doesn't exist!" - exit 1 - fi - FAISS_CONFIGURE_ARGS="${FAISS_CONFIGURE_ARGS} --with-cuda=${CUDA_TOOLKIT_ROOT_DIR} --with-cuda-arch='${FAISS_CUDA_ARCH}'" -else - FAISS_CONFIGURE_ARGS="${FAISS_CONFIGURE_ARGS} --without-cuda" -fi - -if [[ "${FAISS_WITH_MKL}" == "True" ]];then - if [[ ! -d "${MKL_ROOT_DIR}" ]];then - echo "MKL_ROOT_DIR: \"${MKL_ROOT_DIR}\" directory doesn't exist!" - exit 1 - fi - FAISS_CONFIGURE_ARGS="${FAISS_CONFIGURE_ARGS} CPPFLAGS='-DFINTEGER=long -DMKL_ILP64 -m64 -I${MKL_ROOT_DIR}/include' LDFLAGS='-L${MKL_ROOT_DIR}/lib/intel64'" -else - if [[ -n "${LAPACK_PREFIX}" ]];then - if [[ ! -d "${LAPACK_PREFIX}" ]];then - echo "LAPACK_PREFIX: \"${LAPACK_PREFIX}\" directory doesn't exist!" - exit 1 - fi - FAISS_CONFIGURE_ARGS="${FAISS_CONFIGURE_ARGS} LDFLAGS=-L${LAPACK_PREFIX}/lib" - fi - if [[ -n "${OPENBLAS_PREFIX}" ]];then - if [[ ! -d "${OPENBLAS_PREFIX}" ]];then - echo "OPENBLAS_PREFIX: \"${OPENBLAS_PREFIX}\" directory doesn't exist!" - exit 1 - fi - FAISS_CONFIGURE_ARGS="${FAISS_CONFIGURE_ARGS} LDFLAGS=-L${OPENBLAS_PREFIX}/lib" - fi -fi - -cd ${FAISS_SOURCE_DIR} - -sh -c "./configure ${FAISS_CONFIGURE_ARGS}" - -# compile and build -make -j8 || exit 1 -make install || exit 1