diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 6f3ac06b0744560d315f2fe0f683116f1545a1bf..8d61f970b31bfcce9084e122b57f674eb5c0f6c3 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -91,6 +91,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-482 - Change search stream transport to unary in grpc - MS-487 - Define metric type in CreateTable - MS-488 - Improve code format in scheduler +- MS-495 - cmake: integrated knowhere ## New Feature - MS-343 - Implement ResourceMgr diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 08898aacdfec90582eeb25efeaa03222295ef684..16e8041c290f3079177850672ba4d3c1b31b7af8 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -31,6 +31,7 @@ if(CMAKE_BUILD_TYPE STREQUAL "Release") set(BUILD_TYPE "release") else() set(BUILD_TYPE "debug") + SET(CMAKE_VERBOSE_MAKEFILE on) endif() message(STATUS "Build type = ${BUILD_TYPE}") @@ -70,21 +71,21 @@ set(MILVUS_BINARY_DIR ${PROJECT_BINARY_DIR}) find_package(CUDA) set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES -arch sm_60 --expt-extended-lambda") set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g") -message("CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR}") -message("CUDA_NVCC_FLAGS=${CUDA_NVCC_FLAGS}") +message(STATUS "CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR}") +message(STATUS "CUDA_NVCC_FLAGS=${CUDA_NVCC_FLAGS}") set(CMAKE_CXX_STANDARD 14) set(CMAKE_CXX_STANDARD_REQUIRED on) if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)") - message("building milvus_engine on x86 architecture") + message(STATUS "building milvus_engine on x86 architecture") set(MILVUS_BUILD_ARCH x86_64) elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "(ppc)") - message("building milvus_engine on ppc architecture") + message(STATUS "building milvus_engine on ppc architecture") set(MILVUS_BUILD_ARCH ppc64le) else() - message("unknown processor type") - message("CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}") + message(WARNING "unknown processor type") + message(WARNING "CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}") set(MILVUS_BUILD_ARCH unknown) endif() @@ -94,6 +95,8 @@ else() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -DELPP_THREAD_SAFE -fopenmp") endif() +set(ALLOW_DUPLICATE_CUSTOM_TARGETS TRUE) + set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake") include(ExternalProject) @@ -104,14 +107,6 @@ include(ThirdPartyPackages) include_directories(${MILVUS_SOURCE_DIR}) link_directories(${MILVUS_BINARY_DIR}) -if (NOT DEFINED KNOWHERE_BUILD_DIR) - message(FATAL_ERROR "You must set environment variable KNOWHERE_BUILD_DIR") -endif() -message(STATUS "Build with ${KNOWHERE_BUILD_DIR}") -include_directories(${KNOWHERE_BUILD_DIR}/include) -include_directories(${KNOWHERE_BUILD_DIR}/include/SPTAG/AnnService) -link_directories(${KNOWHERE_BUILD_DIR}/lib) - ## Following should be check set(MILVUS_ENGINE_INCLUDE ${PROJECT_SOURCE_DIR}/include) @@ -119,7 +114,7 @@ set(MILVUS_ENGINE_SRC ${PROJECT_SOURCE_DIR}/src) add_compile_definitions(PROFILER=${PROFILER}) -message("MILVUS_ENABLE_PROFILING = ${MILVUS_ENABLE_PROFILING}") +message(STATUS "MILVUS_ENABLE_PROFILING = ${MILVUS_ENABLE_PROFILING}") if (MILVUS_ENABLE_PROFILING STREQUAL "ON") ADD_DEFINITIONS(-DMILVUS_ENABLE_PROFILING) endif() diff --git a/cpp/build.sh b/cpp/build.sh index b2dba3c87086d294d4d24528e1b85cbd51b2db36..aa41b7d3186e840af4872614a0e6907e16e117ae 100755 --- a/cpp/build.sh +++ b/cpp/build.sh @@ -83,9 +83,9 @@ if [[ ! -d cmake_build ]]; then MAKE_CLEAN="ON" fi -pushd `pwd`/src/core -./build.sh ${KNOWHERE_OPTIONS} -popd +#pushd `pwd`/src/core +#./build.sh ${KNOWHERE_OPTIONS} +#popd cd cmake_build diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 80758bfaadd95fb6b8b5a2cd4c605981054b76e7..387693a30c9c831999d266d70d7f43328e123ac0 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -4,6 +4,13 @@ # Proprietary and confidential. #------------------------------------------------------------------------------- +add_subdirectory(core) + +set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE) +foreach(dir ${CORE_INCLUDE_DIRS}) + include_directories(${dir}) +endforeach() + aux_source_directory(cache cache_files) aux_source_directory(config config_files) aux_source_directory(server server_files) @@ -87,12 +94,6 @@ set(client_grpc_lib set(third_party_libs knowhere - SPTAGLibStatic - arrow - jemalloc_pic - faiss - openblas - lapack easyloggingpp sqlite ${client_grpc_lib} @@ -115,15 +116,6 @@ set(third_party_libs cudart ) -if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON") - set(third_party_libs ${third_party_libs} - ${MKL_LIBS}) -else() - set(third_party_libs ${third_party_libs} - lapack - openblas) -endif() - if (MILVUS_ENABLE_PROFILING STREQUAL "ON") set(third_party_libs ${third_party_libs} gperftools @@ -171,13 +163,6 @@ set(server_libs set(knowhere_libs knowhere - SPTAGLibStatic - arrow - jemalloc_pic - faiss - openblas - lapack - tbb ) add_executable(milvus_server @@ -195,8 +180,8 @@ target_link_libraries(milvus_server ${server_libs} ${knowhere_libs} ${third_part install(TARGETS milvus_server DESTINATION bin) install(FILES - ${KNOWHERE_BUILD_DIR}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}tbb${CMAKE_SHARED_LIBRARY_SUFFIX} - ${KNOWHERE_BUILD_DIR}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}tbb${CMAKE_SHARED_LIBRARY_SUFFIX}.2 + ${CMAKE_SOURCE_DIR}/src/core/thirdparty/tbb/${CMAKE_SHARED_LIBRARY_PREFIX}tbb${CMAKE_SHARED_LIBRARY_SUFFIX} + ${CMAKE_SOURCE_DIR}/src/core/thirdparty/tbb/${CMAKE_SHARED_LIBRARY_PREFIX}tbb${CMAKE_SHARED_LIBRARY_SUFFIX}.2 ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX} ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3 ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3.2.4 diff --git a/cpp/src/core/CMakeLists.txt b/cpp/src/core/CMakeLists.txt index 1024a71e71ecc029571b2fd9943ce6348e6c5fc9..890c7ed4d1483c379ad59f373b38f131f1ca6b80 100644 --- a/cpp/src/core/CMakeLists.txt +++ b/cpp/src/core/CMakeLists.txt @@ -5,6 +5,7 @@ #------------------------------------------------------------------------------- cmake_minimum_required(VERSION 3.14) +message(STATUS "---------------core--------------") message(STATUS "Building using CMake version: ${CMAKE_VERSION}") set(KNOWHERE_VERSION "0.1.0") @@ -38,18 +39,16 @@ endif() MESSAGE(STATUS "CMAKE_CXX_FLAGS" ${CMAKE_CXX_FLAGS}) find_package(CUDA) -#set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES --expt-extended-lambda") -#set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g") if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)") - message("building milvus_engine on x86 architecture") + message(STATUS "building milvus_engine on x86 architecture") set(KNOWHERE_BUILD_ARCH x86_64) elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "(ppc)") - message("building milvus_engine on ppc architecture") + message(STATUS "building milvus_engine on ppc architecture") set(KNOWHERE_BUILD_ARCH ppc64le) else() - message("unknown processor type") - message("CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}") + message(WARNING "unknown processor type") + message(WARNING "CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}") set(KNOWHERE_BUILD_ARCH unknown) endif() @@ -60,14 +59,21 @@ else() endif() message(STATUS "Build type = ${BUILD_TYPE}") -set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake") +set(CORE_SOURCE_DIR ${PROJECT_SOURCE_DIR}) +set(CORE_BINARY_DIR ${PROJECT_BINARY_DIR}) +message(STATUS "Core source dir: ${PROJECT_SOURCE_DIR}") +message(STATUS "Core binary dir: ${PROJECT_BINARY_DIR}") +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CORE_SOURCE_DIR}/cmake") + include(ExternalProject) -include(DefineOptions) -include(BuildUtils) -include(ThirdPartyPackages) +include(DefineOptionsCore) +include(BuildUtilsCore) +include(ThirdPartyPackagesCore) add_subdirectory(src) +set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE) + if(BUILD_UNIT_TEST STREQUAL "ON") add_subdirectory(test) endif() diff --git a/cpp/src/core/cmake/BuildUtils.cmake b/cpp/src/core/cmake/BuildUtilsCore.cmake similarity index 99% rename from cpp/src/core/cmake/BuildUtils.cmake rename to cpp/src/core/cmake/BuildUtilsCore.cmake index 2d778bf70368d3dd034a83f0fadad6011d68e99e..ce798c4bb096e470757895b5c3cfd063f9842d13 100644 --- a/cpp/src/core/cmake/BuildUtils.cmake +++ b/cpp/src/core/cmake/BuildUtilsCore.cmake @@ -75,7 +75,7 @@ function(ExternalProject_Use_Cache project_name package_file install_path) "Extracting ${package_file} to ${install_path}" COMMAND ${CMAKE_COMMAND} -E tar xzvf ${package_file} ${install_path} - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + WORKING_DIRECTORY ${CORE_BINARY_DIR} ) ExternalProject_Add_StepTargets(${project_name} extract) diff --git a/cpp/src/core/cmake/DefineOptions.cmake b/cpp/src/core/cmake/DefineOptionsCore.cmake similarity index 98% rename from cpp/src/core/cmake/DefineOptions.cmake rename to cpp/src/core/cmake/DefineOptionsCore.cmake index 589e2d9ae8456a2f4e27b590479a408e5eb12613..e20d3d77abb09bbfdcf402596d2b817efefca1f1 100644 --- a/cpp/src/core/cmake/DefineOptions.cmake +++ b/cpp/src/core/cmake/DefineOptionsCore.cmake @@ -104,7 +104,7 @@ macro(config_summary) message(STATUS " Source directory: ${CMAKE_CURRENT_SOURCE_DIR}") if(${CMAKE_EXPORT_COMPILE_COMMANDS}) message( - STATUS " Compile commands: ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json") + STATUS " Compile commands: ${CORE_BINARY_DIR}/compile_commands.json") endif() foreach(category ${KNOWHERE_OPTION_CATEGORIES}) diff --git a/cpp/src/core/cmake/ThirdPartyPackages.cmake b/cpp/src/core/cmake/ThirdPartyPackagesCore.cmake similarity index 70% rename from cpp/src/core/cmake/ThirdPartyPackages.cmake rename to cpp/src/core/cmake/ThirdPartyPackagesCore.cmake index 4a5e47f9534b4adfc5de8a5efea280b7e48631ec..fad08690218176a7c37dda1289482ff5b6b02bd5 100644 --- a/cpp/src/core/cmake/ThirdPartyPackages.cmake +++ b/cpp/src/core/cmake/ThirdPartyPackagesCore.cmake @@ -6,7 +6,6 @@ # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an @@ -18,7 +17,6 @@ set(KNOWHERE_THIRDPARTY_DEPENDENCIES ARROW - BOOST FAISS GTest LAPACK @@ -53,11 +51,8 @@ endmacro() macro(resolve_dependency DEPENDENCY_NAME) if (${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO") #message(STATUS "Finding ${DEPENDENCY_NAME} package") -# find_package(${DEPENDENCY_NAME} QUIET) -# if (NOT ${DEPENDENCY_NAME}_FOUND) #message(STATUS "${DEPENDENCY_NAME} package not found") build_dependency(${DEPENDENCY_NAME}) -# endif () elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "BUNDLED") build_dependency(${DEPENDENCY_NAME}) elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "SYSTEM") @@ -120,7 +115,7 @@ endif (UNIX) # ---------------------------------------------------------------------- # thirdparty directory -set(THIRDPARTY_DIR "${CMAKE_SOURCE_DIR}/thirdparty") +set(THIRDPARTY_DIR "${CORE_SOURCE_DIR}/thirdparty") # ---------------------------------------------------------------------- # JFrog @@ -202,14 +197,6 @@ endif() set(MAKE_BUILD_ARGS "-j8") -## Using make -j in sub-make is fragile -## see discussion https://github.com/apache/KNOWHERE/pull/2779 -#if(${CMAKE_GENERATOR} MATCHES "Makefiles") -# set(MAKE_BUILD_ARGS "") -#else() -# # limit the maximum number of jobs for ninja -# set(MAKE_BUILD_ARGS "-j4") -#endif() # ---------------------------------------------------------------------- # Find pthreads @@ -243,22 +230,11 @@ foreach(_VERSION_ENTRY ${TOOLCHAIN_VERSIONS_TXT}) set(${_LIB_NAME} "${_LIB_VERSION}") endforeach() -if(DEFINED ENV{KNOWHERE_BOOST_URL}) - set(BOOST_SOURCE_URL "$ENV{KNOWHERE_BOOST_URL}") -else() - string(REPLACE "." "_" BOOST_VERSION_UNDERSCORES ${BOOST_VERSION}) - set(BOOST_SOURCE_URL - "https://dl.bintray.com/boostorg/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDERSCORES}.tar.gz" - ) -endif() - if(DEFINED ENV{KNOWHERE_FAISS_URL}) set(FAISS_SOURCE_URL "$ENV{KNOWHERE_FAISS_URL}") else() set(FAISS_SOURCE_URL "http://192.168.1.105:6060/jinhai/faiss/-/archive/${FAISS_VERSION}/faiss-${FAISS_VERSION}.tar.gz") -# set(FAISS_SOURCE_URL "https://github.com/facebookresearch/faiss/archive/${FAISS_VERSION}.tar.gz") -# set(FAISS_SOURCE_URL "${CMAKE_SOURCE_DIR}/thirdparty/faiss-1.5.3") - message(STATUS ${FAISS_SOURCE_URL}) + message(STATUS "FAISS URL = ${FAISS_SOURCE_URL}") endif() # set(FAISS_MD5 "a589663865a8558205533c8ac414278c") # set(FAISS_MD5 "57da9c4f599cc8fa4260488b1c96e1cc") # commit-id 6dbdf75987c34a2c853bd172ea0d384feea8358c @@ -297,28 +273,18 @@ set(OPENBLAS_MD5 "8a110a25b819a4b94e8a9580702b6495") # ---------------------------------------------------------------------- # ARROW +set(ARROW_PREFIX "${CORE_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep/cpp") macro(build_arrow) message(STATUS "Building Apache ARROW-${ARROW_VERSION} from source") - set(ARROW_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep/cpp") set(ARROW_STATIC_LIB_NAME arrow) -# set(PARQUET_STATIC_LIB_NAME parquet) - # set(ARROW_CUDA_STATIC_LIB_NAME arrow_cuda) - set(ARROW_STATIC_LIB + set(ARROW_STATIC_LIB "${ARROW_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${ARROW_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" ) -# set(PARQUET_STATIC_LIB -# "${ARROW_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${PARQUET_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" -# ) - # set(ARROW_CUDA_STATIC_LIB - # "${ARROW_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${ARROW_CUDA_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" - # ) - set(ARROW_INCLUDE_DIR "${ARROW_PREFIX}/include") + set(ARROW_INCLUDE_DIR "${ARROW_PREFIX}/include") set(ARROW_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} - # "-DARROW_THRIFT_URL=${THRIFT_SOURCE_URL}" - #"env ARROW_THRIFT_URL=${THRIFT_SOURCE_URL}" -DARROW_BUILD_STATIC=ON -DARROW_BUILD_SHARED=OFF -DARROW_PARQUET=OFF @@ -328,8 +294,7 @@ macro(build_arrow) -DCMAKE_BUILD_TYPE=Release -DARROW_DEPENDENCY_SOURCE=BUNDLED) #Build all arrow dependencies from source instead of calling find_package first - # set($ENV{ARROW_THRIFT_URL} ${THRIFT_SOURCE_URL}) - + if(USE_JFROG_CACHE STREQUAL "ON") execute_process(COMMAND sh -c "git ls-remote --heads --tags ${ARROW_SOURCE_URL} ${ARROW_VERSION} | cut -f 1" OUTPUT_VARIABLE ARROW_LAST_COMMIT_ID) if(${ARROW_LAST_COMMIT_ID} MATCHES "^[^#][a-z0-9]+") @@ -362,13 +327,13 @@ macro(build_arrow) "${ARROW_STATIC_LIB}" ) - ExternalProject_Create_Cache(arrow_ep ${ARROW_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${ARROW_CACHE_URL}) + ExternalProject_Create_Cache(arrow_ep ${ARROW_CACHE_PACKAGE_PATH} "${CORE_BINARY_DIR}/arrow_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${ARROW_CACHE_URL}) else() file(DOWNLOAD ${ARROW_CACHE_URL} ${ARROW_CACHE_PACKAGE_PATH} STATUS status) list(GET status 0 status_code) message(STATUS "DOWNLOADING FROM ${ARROW_CACHE_URL} TO ${ARROW_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") if (status_code EQUAL 0) - ExternalProject_Use_Cache(arrow_ep ${ARROW_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) + ExternalProject_Use_Cache(arrow_ep ${ARROW_CACHE_PACKAGE_PATH} ${CORE_BINARY_DIR}) endif() endif() else() @@ -402,10 +367,9 @@ macro(build_arrow) set_target_properties(arrow PROPERTIES IMPORTED_LOCATION "${ARROW_STATIC_LIB}" INTERFACE_INCLUDE_DIRECTORIES "${ARROW_INCLUDE_DIR}") - # INTERFACE_LINK_LIBRARIES thrift) - add_dependencies(arrow arrow_ep) + add_dependencies(arrow arrow_ep) - set(JEMALLOC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep-build/jemalloc_ep-prefix/src/jemalloc_ep") + set(JEMALLOC_PREFIX "${CORE_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep-build/jemalloc_ep-prefix/src/jemalloc_ep") add_custom_command(TARGET arrow_ep POST_BUILD COMMAND ${CMAKE_COMMAND} -E make_directory ${ARROW_PREFIX}/lib/ @@ -414,7 +378,7 @@ macro(build_arrow) endmacro() -if(KNOWHERE_WITH_ARROW) +if(KNOWHERE_WITH_ARROW AND NOT TARGET arrow_ep) resolve_dependency(ARROW) @@ -422,191 +386,12 @@ if(KNOWHERE_WITH_ARROW) include_directories(SYSTEM ${ARROW_INCLUDE_DIR}) endif() -# ---------------------------------------------------------------------- -# Add Boost dependencies (code adapted from Apache Kudu (incubating)) - -set(Boost_USE_MULTITHREADED ON) -if(MSVC AND KNOWHERE_USE_STATIC_CRT) - set(Boost_USE_STATIC_RUNTIME ON) -endif() -set(Boost_ADDITIONAL_VERSIONS - "1.70.0" - "1.70" - "1.69.0" - "1.69" - "1.68.0" - "1.68" - "1.67.0" - "1.67" - "1.66.0" - "1.66" - "1.65.0" - "1.65" - "1.64.0" - "1.64" - "1.63.0" - "1.63" - "1.62.0" - "1.61" - "1.61.0" - "1.62" - "1.60.0" - "1.60") - -# TODO -if(KNOWHERE_BOOST_VENDORED) -# system thread serialization wserialization regex - set(BOOST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/boost_ep-prefix/src/boost_ep") - set(BOOST_LIB_DIR "${BOOST_PREFIX}/stage/lib") - set(BOOST_BUILD_LINK "static") - set(BOOST_STATIC_SYSTEM_LIBRARY - "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_system${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - set(BOOST_STATIC_FILESYSTEM_LIBRARY - "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_filesystem${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - set(BOOST_STATIC_SERIALIZATION_LIBRARY - "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_serialization${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - set(BOOST_STATIC_WSERIALIZATION_LIBRARY - "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_wserialization${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - set(BOOST_STATIC_REGEX_LIBRARY - "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_regex${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - set(BOOST_STATIC_THREAD_LIBRARY - "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_thread${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - set(BOOST_SYSTEM_LIBRARY boost_system_static) - set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_static) - set(BOOST_SERIALIZATION_LIBRARY boost_serialization_static) - set(BOOST_WSERIALIZATION_LIBRARY boost_wserialization_static) - set(BOOST_REGEX_LIBRARY boost_regex_static) - set(BOOST_THREAD_LIBRARY boost_thread_static) - - if(KNOWHERE_BOOST_HEADER_ONLY) - set(BOOST_BUILD_PRODUCTS) - set(BOOST_CONFIGURE_COMMAND "") - set(BOOST_BUILD_COMMAND "") - else() - set(BOOST_BUILD_PRODUCTS ${BOOST_STATIC_SYSTEM_LIBRARY} - ${BOOST_STATIC_FILESYSTEM_LIBRARY} ${BOOST_STATIC_SERIALIZATION_LIBRARY} - ${BOOST_STATIC_WSERIALIZATION_LIBRARY} ${BOOST_STATIC_REGEX_LIBRARY} - ${BOOST_STATIC_THREAD_LIBRARY}) - set(BOOST_CONFIGURE_COMMAND "./bootstrap.sh" "--prefix=${BOOST_PREFIX}" - "--with-libraries=filesystem,serialization,wserialization,system,thread,regex") - if("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG") - set(BOOST_BUILD_VARIANT "debug") - else() - set(BOOST_BUILD_VARIANT "release") - endif() - set(BOOST_BUILD_COMMAND - "./b2" - "link=${BOOST_BUILD_LINK}" - "variant=${BOOST_BUILD_VARIANT}" - "cxxflags=-fPIC") - - add_thirdparty_lib(boost_system STATIC_LIB "${BOOST_STATIC_SYSTEM_LIBRARY}") - - add_thirdparty_lib(boost_filesystem STATIC_LIB "${BOOST_STATIC_FILESYSTEM_LIBRARY}") - - add_thirdparty_lib(boost_serialization STATIC_LIB "${BOOST_STATIC_SERIALIZATION_LIBRARY}") - - add_thirdparty_lib(boost_wserialization STATIC_LIB "${BOOST_STATIC_WSERIALIZATION_LIBRARY}") - - add_thirdparty_lib(boost_regex STATIC_LIB "${BOOST_STATIC_REGEX_LIBRARY}") - - add_thirdparty_lib(boost_thread STATIC_LIB "${BOOST_STATIC_THREAD_LIBRARY}") - - set(KNOWHERE_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY} ${BOOST_SERIALIZATION_LIBRARY} - ${BOOST_WSERIALIZATION_LIBRARY} ${BOOST_REGEX_LIBRARY} ${BOOST_THREAD_LIBRARY}) - endif() - externalproject_add(boost_ep - URL - ${BOOST_SOURCE_URL} - BUILD_BYPRODUCTS - ${BOOST_BUILD_PRODUCTS} - BUILD_IN_SOURCE - 1 - CONFIGURE_COMMAND - ${BOOST_CONFIGURE_COMMAND} - BUILD_COMMAND - ${BOOST_BUILD_COMMAND} - INSTALL_COMMAND - "" - ${EP_LOG_OPTIONS}) - set(Boost_INCLUDE_DIR "${BOOST_PREFIX}") - set(Boost_INCLUDE_DIRS "${BOOST_INCLUDE_DIR}") - add_dependencies(boost_system_static boost_ep) - add_dependencies(boost_filesystem_static boost_ep) - add_dependencies(boost_serialization_static boost_ep) - add_dependencies(boost_wserialization_static boost_ep) - add_dependencies(boost_regex_static boost_ep) - add_dependencies(boost_thread_static boost_ep) - -#else() -# if(MSVC) -# # disable autolinking in boost -# add_definitions(-DBOOST_ALL_NO_LIB) -# endif() - -# if(DEFINED ENV{BOOST_ROOT} OR DEFINED BOOST_ROOT) -# # In older versions of CMake (such as 3.2), the system paths for Boost will -# # In older versions of CMake (such as 3.2), the system paths for Boost will -# # be looked in first even if we set $BOOST_ROOT or pass -DBOOST_ROOT -# set(Boost_NO_SYSTEM_PATHS ON) -# endif() - -# if(KNOWHERE_BOOST_USE_SHARED) -# # Find shared Boost libraries. -# set(Boost_USE_STATIC_LIBS OFF) -# set(BUILD_SHARED_LIBS_KEEP ${BUILD_SHARED_LIBS}) -# set(BUILD_SHARED_LIBS ON) -# -# if(MSVC) -# # force all boost libraries to dynamic link -# add_definitions(-DBOOST_ALL_DYN_LINK) -# endif() -# -# if(KNOWHERE_BOOST_HEADER_ONLY) -# find_package(Boost REQUIRED) -# else() -# find_package(Boost COMPONENTS serialization system filesystem REQUIRED) -# set(BOOST_SYSTEM_LIBRARY Boost::system) -# set(BOOST_FILESYSTEM_LIBRARY Boost::filesystem) -# set(BOOST_SERIALIZATION_LIBRARY Boost::serialization) -# set(KNOWHERE_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY}) -# endif() -# set(BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS_KEEP}) -# unset(BUILD_SHARED_LIBS_KEEP) -# else() -# # Find static boost headers and libs -# # TODO Differentiate here between release and debug builds -# set(Boost_USE_STATIC_LIBS ON) -# if(KNOWHERE_BOOST_HEADER_ONLY) -# find_package(Boost REQUIRED) -# else() -# find_package(Boost COMPONENTS serialization system filesystem REQUIRED) -# set(BOOST_SYSTEM_LIBRARY Boost::system) -# set(BOOST_FILESYSTEM_LIBRARY Boost::filesystem) -# set(BOOST_SERIALIZATION_LIBRARY Boost::serialization) -# set(KNOWHERE_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY}) -# endif() -# endif() -endif() - -#message(STATUS "Boost include dir: " ${Boost_INCLUDE_DIR}) -#message(STATUS "Boost libraries: " ${Boost_LIBRARIES}) - -include_directories(SYSTEM ${Boost_INCLUDE_DIR}) -link_directories(SYSTEM ${BOOST_LIB_DIR}) - # ---------------------------------------------------------------------- # OpenBLAS macro(build_openblas) message(STATUS "Building OpenBLAS-${OPENBLAS_VERSION} from source") - set(OPENBLAS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/openblas_ep-prefix/src/openblas_ep") + set(OPENBLAS_PREFIX "${CORE_BINARY_DIR}/openblas_ep-prefix/src/openblas_ep") set(OPENBLAS_INCLUDE_DIR "${OPENBLAS_PREFIX}/include") set(OPENBLAS_STATIC_LIB "${OPENBLAS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}openblas${CMAKE_STATIC_LIBRARY_SUFFIX}") @@ -639,13 +424,13 @@ macro(build_openblas) BUILD_BYPRODUCTS ${OPENBLAS_STATIC_LIB}) - ExternalProject_Create_Cache(openblas_ep ${OPENBLAS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/openblas_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${OPENBLAS_CACHE_URL}) + ExternalProject_Create_Cache(openblas_ep ${OPENBLAS_CACHE_PACKAGE_PATH} "${CORE_BINARY_DIR}/openblas_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${OPENBLAS_CACHE_URL}) else() file(DOWNLOAD ${OPENBLAS_CACHE_URL} ${OPENBLAS_CACHE_PACKAGE_PATH} STATUS status) list(GET status 0 status_code) message(STATUS "DOWNLOADING FROM ${OPENBLAS_CACHE_URL} TO ${OPENBLAS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") if (status_code EQUAL 0) - ExternalProject_Use_Cache(openblas_ep ${OPENBLAS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) + ExternalProject_Use_Cache(openblas_ep ${OPENBLAS_CACHE_PACKAGE_PATH} ${CORE_BINARY_DIR}) endif() endif() else() @@ -678,20 +463,12 @@ macro(build_openblas) add_dependencies(openblas openblas_ep) endmacro() -#if(KNOWHERE_WITH_OPENBLAS) -# resolve_dependency(OpenBLAS) -# -# get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES) -# include_directories(SYSTEM "${OPENBLAS_INCLUDE_DIR}") -# link_directories(SYSTEM ${OPENBLAS_PREFIX}/lib) -#endif() - # ---------------------------------------------------------------------- # LAPACK macro(build_lapack) message(STATUS "Building LAPACK-${LAPACK_VERSION} from source") - set(LAPACK_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/lapack_ep-prefix/src/lapack_ep") + set(LAPACK_PREFIX "${CORE_BINARY_DIR}/lapack_ep-prefix/src/lapack_ep") set(LAPACK_INCLUDE_DIR "${LAPACK_PREFIX}/include") set(LAPACK_STATIC_LIB "${LAPACK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}lapack${CMAKE_STATIC_LIBRARY_SUFFIX}") @@ -723,13 +500,13 @@ macro(build_lapack) BUILD_BYPRODUCTS ${LAPACK_STATIC_LIB}) - ExternalProject_Create_Cache(lapack_ep ${LAPACK_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/lapack_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${LAPACK_CACHE_URL}) + ExternalProject_Create_Cache(lapack_ep ${LAPACK_CACHE_PACKAGE_PATH} "${CORE_BINARY_DIR}/lapack_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${LAPACK_CACHE_URL}) else() file(DOWNLOAD ${LAPACK_CACHE_URL} ${LAPACK_CACHE_PACKAGE_PATH} STATUS status) list(GET status 0 status_code) message(STATUS "DOWNLOADING FROM ${LAPACK_CACHE_URL} TO ${LAPACK_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") if (status_code EQUAL 0) - ExternalProject_Use_Cache(lapack_ep ${LAPACK_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) + ExternalProject_Use_Cache(lapack_ep ${LAPACK_CACHE_PACKAGE_PATH} ${CORE_BINARY_DIR}) endif() endif() else() @@ -756,14 +533,6 @@ macro(build_lapack) add_dependencies(lapack lapack_ep) endmacro() -#if(KNOWHERE_WITH_LAPACK) -# resolve_dependency(LAPACK) -# -# get_target_property(LAPACK_INCLUDE_DIR lapack INTERFACE_INCLUDE_DIRECTORIES) -# include_directories(SYSTEM "${LAPACK_INCLUDE_DIR}") -# link_directories(SYSTEM ${LAPACK_PREFIX}/lib) -#endif() - # ---------------------------------------------------------------------- # Google gtest @@ -780,7 +549,7 @@ macro(build_gtest) -Wno-ignored-attributes) endif() - set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep") + set(GTEST_PREFIX "${CORE_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep") set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include") set(GTEST_STATIC_LIB "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}") @@ -822,13 +591,13 @@ macro(build_gtest) ${GTEST_CMAKE_ARGS} ${EP_LOG_OPTIONS}) - ExternalProject_Create_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${GTEST_CACHE_URL}) + ExternalProject_Create_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} "${CORE_BINARY_DIR}/googletest_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${GTEST_CACHE_URL}) else() file(DOWNLOAD ${GTEST_CACHE_URL} ${GTEST_CACHE_PACKAGE_PATH} STATUS status) list(GET status 0 status_code) message(STATUS "DOWNLOADING FROM ${GTEST_CACHE_URL} TO ${GTEST_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") if (status_code EQUAL 0) - ExternalProject_Use_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) + ExternalProject_Use_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} ${CORE_BINARY_DIR}) endif() endif() else() @@ -871,8 +640,7 @@ macro(build_gtest) endmacro() -if (KNOWHERE_BUILD_TESTS) - #message(STATUS "Resolving gtest dependency") +if (KNOWHERE_BUILD_TESTS AND NOT TARGET googletest_ep) resolve_dependency(GTest) if(NOT GTEST_VENDORED) @@ -889,17 +657,12 @@ endif() macro(build_faiss) message(STATUS "Building FAISS-${FAISS_VERSION} from source") - set(FAISS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/faiss_ep-prefix/src/faiss_ep") + set(FAISS_PREFIX "${CORE_BINARY_DIR}/faiss_ep-prefix/src/faiss_ep") set(FAISS_INCLUDE_DIR "${FAISS_PREFIX}/include") set(FAISS_STATIC_LIB "${FAISS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}faiss${CMAKE_STATIC_LIBRARY_SUFFIX}") - # add_custom_target(faiss_dependencies) - # add_dependencies(faiss_dependencies openblas_ep) - # add_dependencies(faiss_dependencies openblas) - # get_target_property(FAISS_OPENBLAS_LIB_DIR openblas IMPORTED_LOCATION) - # get_filename_component(FAISS_OPENBLAS_LIB "${FAISS_OPENBLAS_LIB_DIR}" DIRECTORY) - + set(FAISS_CONFIGURE_ARGS "--prefix=${FAISS_PREFIX}" "CFLAGS=${EP_C_FLAGS}" @@ -907,13 +670,7 @@ macro(build_faiss) "LDFLAGS=-L${OPENBLAS_PREFIX}/lib -L${LAPACK_PREFIX}/lib -lopenblas -llapack" --without-python) - # if(OPENBLAS_STATIC_LIB) - # set(OPENBLAS_LIBRARY ${OPENBLAS_STATIC_LIB}) - # else() - # set(OPENBLAS_LIBRARY ${OPENBLAS_SHARED_LIB}) - # endif() - # set(FAISS_DEPENDENCIES ${FAISS_DEPENDENCIES} ${OPENBLAS_LIBRARY}) - + if(${KNOWHERE_WITH_FAISS_GPU_VERSION} STREQUAL "ON") set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} "--with-cuda=${CUDA_TOOLKIT_ROOT_DIR}" @@ -924,9 +681,7 @@ macro(build_faiss) endif() if(USE_JFROG_CACHE STREQUAL "ON") -# Check_Last_Modify("${CMAKE_SOURCE_DIR}/thirdparty/faiss_cache_check_lists.txt" "${CMAKE_SOURCE_DIR}" FAISS_LAST_MODIFIED_COMMIT_ID) string(MD5 FAISS_COMBINE_MD5 "${FAISS_MD5}${LAPACK_MD5}${OPENBLAS_MD5}") - # string(MD5 FAISS_COMBINE_MD5 "${FAISS_LAST_MODIFIED_COMMIT_ID}${LAPACK_MD5}${OPENBLAS_MD5}") set(FAISS_CACHE_PACKAGE_NAME "faiss_${FAISS_COMBINE_MD5}.tar.gz") set(FAISS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${FAISS_CACHE_PACKAGE_NAME}") set(FAISS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${FAISS_CACHE_PACKAGE_NAME}") @@ -952,13 +707,13 @@ macro(build_faiss) ExternalProject_Add_StepDependencies(faiss_ep build openblas_ep lapack_ep) - ExternalProject_Create_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/faiss_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${FAISS_CACHE_URL}) + ExternalProject_Create_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} "${CORE_BINARY_DIR}/faiss_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${FAISS_CACHE_URL}) else() file(DOWNLOAD ${FAISS_CACHE_URL} ${FAISS_CACHE_PACKAGE_PATH} STATUS status) list(GET status 0 status_code) message(STATUS "DOWNLOADING FROM ${FAISS_CACHE_URL} TO ${FAISS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") if (status_code EQUAL 0) - ExternalProject_Use_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) + ExternalProject_Use_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} ${CORE_BINARY_DIR}) endif() endif() else() @@ -990,14 +745,10 @@ macro(build_faiss) INTERFACE_LINK_LIBRARIES "openblas;lapack" ) add_dependencies(faiss faiss_ep) - #add_dependencies(faiss openblas_ep) - #add_dependencies(faiss lapack_ep) - #target_link_libraries(faiss ${OPENBLAS_PREFIX}/lib) - #target_link_libraries(faiss ${LAPACK_PREFIX}/lib) endmacro() -if(KNOWHERE_WITH_FAISS) +if(KNOWHERE_WITH_FAISS AND NOT TARGET faiss_ep) resolve_dependency(OpenBLAS) get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES) @@ -1012,8 +763,5 @@ if(KNOWHERE_WITH_FAISS) resolve_dependency(FAISS) get_target_property(FAISS_INCLUDE_DIR faiss INTERFACE_INCLUDE_DIRECTORIES) include_directories(SYSTEM "${FAISS_INCLUDE_DIR}") -# include_directories(SYSTEM "${CMAKE_CURRENT_BINARY_DIR}/faiss_ep-prefix/src/") -# link_directories(SYSTEM ${FAISS_PREFIX}/) link_directories(SYSTEM ${FAISS_PREFIX}/lib/) -# link_directories(SYSTEM ${FAISS_PREFIX}/gpu/) endif() diff --git a/cpp/src/core/src/CMakeLists.txt b/cpp/src/core/src/CMakeLists.txt index 5b119f8b2b31872ea7076ce4254b6ab324eca6af..a51a62b44ad48c2887e29e13965a0f7d8d1f5829 100644 --- a/cpp/src/core/src/CMakeLists.txt +++ b/cpp/src/core/src/CMakeLists.txt @@ -1,16 +1,16 @@ -set(TBB_DIR ${CMAKE_SOURCE_DIR}/thirdparty/tbb) +set(TBB_DIR ${CORE_SOURCE_DIR}/thirdparty/tbb) set(TBB_LIBRARIES ${TBB_DIR}/libtbb.so) include_directories(${TBB_DIR}/include) include_directories(${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) link_directories(${CUDA_TOOLKIT_ROOT_DIR}/lib64) -include_directories(${CMAKE_SOURCE_DIR}/include) -include_directories(${CMAKE_SOURCE_DIR}/thirdparty) -include_directories(${CMAKE_SOURCE_DIR}/thirdparty/SPTAG/AnnService) -include_directories(${CMAKE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include) +include_directories(${CORE_SOURCE_DIR}/include) +include_directories(${CORE_SOURCE_DIR}/thirdparty) +include_directories(${CORE_SOURCE_DIR}/thirdparty/SPTAG/AnnService) +include_directories(${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include) -set(SPTAG_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../thirdparty/SPTAG) +set(SPTAG_SOURCE_DIR ${CORE_SOURCE_DIR}/thirdparty/SPTAG) file(GLOB HDR_FILES ${SPTAG_SOURCE_DIR}/AnnService/inc/Core/*.h ${SPTAG_SOURCE_DIR}/AnnService/inc/Core/Common/*.h @@ -24,9 +24,9 @@ file(GLOB SRC_FILES ${SPTAG_SOURCE_DIR}/AnnService/src/Core/KDT/*.cpp ${SPTAG_SOURCE_DIR}/AnnService/src/Helper/*.cpp) -#add_library(SPTAGLib SHARED ${SRC_FILES} ${HDR_FILES}) -#target_link_libraries(SPTAGLib ${TBB_LIBRARIES}) -add_library(SPTAGLibStatic STATIC ${SRC_FILES} ${HDR_FILES}) +if(NOT TARGET SPTAGLibStatic) + add_library(SPTAGLibStatic STATIC ${SRC_FILES} ${HDR_FILES}) +endif() set(external_srcs knowhere/adapter/sptag.cpp @@ -51,14 +51,13 @@ set(index_srcs ) set(depend_libs -# libtcmalloc.a SPTAGLibStatic ${TBB_LIBRARIES} faiss openblas lapack arrow - jemalloc_pic + ${ARROW_PREFIX}/lib/libjemalloc_pic.a cudart cublas gomp @@ -66,30 +65,19 @@ set(depend_libs pthread ) -add_library( - knowhere STATIC - ${external_srcs} - ${index_srcs} -) -#target_compile_options(knowhere PUBLIC "-fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free") +if(NOT TARGET knowhere) + add_library( + knowhere STATIC + ${external_srcs} + ${index_srcs} + ) +endif() target_link_libraries( knowhere ${depend_libs} ) -#add_library( -# knowhereS SHARED -# ${external_srcs} -# ${index_srcs} -#) -# -#target_link_libraries( -# knowhereS -## ${TBB_LIBRARIES} -# ${depend_libs} -#) - INSTALL(TARGETS knowhere SPTAGLibStatic @@ -98,7 +86,6 @@ INSTALL(TARGETS INSTALL(FILES ${ARROW_STATIC_LIB} -# ${PARQUET_STATIC_LIB} ${ARROW_PREFIX}/lib/libjemalloc_pic.a ${FAISS_STATIC_LIB} ${LAPACK_STATIC_LIB} @@ -112,24 +99,35 @@ INSTALL(FILES ${OPENBLAS_REAL_STATIC_LIB} DESTINATION lib ) -INSTALL(FILES ${CMAKE_SOURCE_DIR}/thirdparty/tbb/libtbb.so.2 -# RENAME "libtbb.so.2" +INSTALL(FILES ${CORE_SOURCE_DIR}/thirdparty/tbb/libtbb.so.2 DESTINATION lib ) -INSTALL(FILES ${CMAKE_SOURCE_DIR}/thirdparty/tbb/libtbb.so -# RENAME "libtbb.so" +INSTALL(FILES ${CORE_SOURCE_DIR}/thirdparty/tbb/libtbb.so DESTINATION lib ) +set(CORE_INCLUDE_DIRS + ${CORE_SOURCE_DIR}/include + ${CORE_SOURCE_DIR}/thirdparty + ${CORE_SOURCE_DIR}/thirdparty/SPTAG/AnnService + ${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include + ${ARROW_INCLUDE_DIR} + ${FAISS_INCLUDE_DIR} + ${OPENBLAS_INCLUDE_DIR} + ${LAPACK_INCLUDE_DIR} + ${CORE_SOURCE_DIR}/thirdparty/tbb/include + ) + +set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE) + INSTALL(DIRECTORY - ${CMAKE_SOURCE_DIR}/include/knowhere - ${CMAKE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons - ${CMAKE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons_ext + ${CORE_SOURCE_DIR}/include/knowhere + ${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons + ${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons_ext ${ARROW_INCLUDE_DIR}/arrow -# ${ARROW_INCLUDE_DIR}/parquet ${FAISS_PREFIX}/include/faiss ${OPENBLAS_INCLUDE_DIR}/ - ${CMAKE_SOURCE_DIR}/thirdparty/tbb/include/tbb + ${CORE_SOURCE_DIR}/thirdparty/tbb/include/tbb DESTINATION include) diff --git a/cpp/src/core/test/CMakeLists.txt b/cpp/src/core/test/CMakeLists.txt index fbbf151264ec9a187d4e44d14b5c4df51f0e9b5e..ba3f0d65bdbcfba2d6cf84860ce6be75b0940d26 100644 --- a/cpp/src/core/test/CMakeLists.txt +++ b/cpp/src/core/test/CMakeLists.txt @@ -1,17 +1,19 @@ -include_directories(${CMAKE_SOURCE_DIR}/thirdparty) -include_directories(${CMAKE_SOURCE_DIR}/thirdparty/SPTAG/AnnService) -include_directories(${CMAKE_SOURCE_DIR}/include) -include_directories(${CMAKE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include) +include_directories(${CORE_SOURCE_DIR}/thirdparty) +include_directories(${CORE_SOURCE_DIR}/thirdparty/SPTAG/AnnService) +include_directories(${CORE_SOURCE_DIR}/include) +include_directories(${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include) include_directories(/usr/local/cuda/include) link_directories(/usr/local/cuda/lib64) -link_directories(${CMAKE_SOURCE_DIR}/thirdparty/tbb) +link_directories(${CORE_SOURCE_DIR}/thirdparty/tbb) set(unittest_libs gtest gmock gtest_main gmock_main) +message(STATUS "arrow prefix: ${ARROW_PREFIX}") +message(STATUS "libjemalloc_pic path: ${ARROW_PREFIX}/lib/libjemalloc_pic.a") set(depend_libs faiss openblas lapack - arrow jemalloc_pic + arrow "${ARROW_PREFIX}/lib/libjemalloc_pic.a" tbb ) @@ -22,57 +24,55 @@ set(basic_libs # -#<<<<<<<<<< set(ivf_srcs - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/common/exception.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/common/timer.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp + ${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp + ${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp + ${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp utils.cpp ) -add_executable(test_ivf test_ivf.cpp ${ivf_srcs}) +if(NOT TARGET test_ivf) + add_executable(test_ivf test_ivf.cpp ${ivf_srcs}) +endif() target_link_libraries(test_ivf ${depend_libs} ${unittest_libs} ${basic_libs}) -#<<<<<<<<<< # -#<<<<<<<<<< set(idmap_srcs - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/common/exception.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/common/timer.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp + ${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp + ${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp + ${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp utils.cpp ) -add_executable(test_idmap test_idmap.cpp ${idmap_srcs}) +if(NOT TARGET test_idmap) + add_executable(test_idmap test_idmap.cpp ${idmap_srcs}) +endif() target_link_libraries(test_idmap ${depend_libs} ${unittest_libs} ${basic_libs}) -#<<<<<<<<<< - # -#<<<<<<<<<< set(kdt_srcs - ${CMAKE_SOURCE_DIR}/src/knowhere/index/preprocessor/normalize.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/kdt_parameters.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/cpu_kdt_rng.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/adapter/sptag.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/common/exception.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/adapter/arrow.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/common/timer.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/preprocessor/normalize.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/kdt_parameters.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cpu_kdt_rng.cpp + ${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp + ${CORE_SOURCE_DIR}/src/knowhere/adapter/sptag.cpp + ${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp + ${CORE_SOURCE_DIR}/src/knowhere/adapter/arrow.cpp + ${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp utils.cpp ) -add_executable(test_kdt test_kdt.cpp ${kdt_srcs}) +if(NOT TARGET test_kdt) + add_executable(test_kdt test_kdt.cpp ${kdt_srcs}) +endif() target_link_libraries(test_kdt SPTAGLibStatic ${depend_libs} ${unittest_libs} ${basic_libs}) -#<<<<<<<<<< - add_subdirectory(faiss_ori) add_subdirectory(test_nsg) diff --git a/cpp/src/core/test/faiss_ori/CMakeLists.txt b/cpp/src/core/test/faiss_ori/CMakeLists.txt index a6b20a237304082ffb01de0bc539be2f5146db54..2f46d25d40b2ff3a57ddbe249c33c3dfee1e48ff 100644 --- a/cpp/src/core/test/faiss_ori/CMakeLists.txt +++ b/cpp/src/core/test/faiss_ori/CMakeLists.txt @@ -1,15 +1,15 @@ -include_directories(${CMAKE_SOURCE_DIR}/thirdparty) -include_directories(${CMAKE_SOURCE_DIR}/include) +include_directories(${CORE_SOURCE_DIR}/thirdparty) +include_directories(${CORE_SOURCE_DIR}/include) include_directories(/usr/local/cuda/include) link_directories(/usr/local/cuda/lib64) -link_directories(${CMAKE_SOURCE_DIR}/thirdparty/tbb) +link_directories(${CORE_SOURCE_DIR}/thirdparty/tbb) set(unittest_libs gtest gmock gtest_main gmock_main) set(depend_libs faiss openblas lapack - arrow jemalloc_pic + arrow ${ARROW_PREFIX}/lib/libjemalloc_pic.a tbb ) @@ -20,7 +20,7 @@ set(basic_libs # -#<<<<<<<<<< -add_executable(test_gpu gpuresource_test.cpp) +if(NOT TARGET test_gpu) + add_executable(test_gpu gpuresource_test.cpp) +endif() target_link_libraries(test_gpu ${depend_libs} ${unittest_libs} ${basic_libs}) -#<<<<<<<<<< diff --git a/cpp/src/core/test/test_nsg/CMakeLists.txt b/cpp/src/core/test/test_nsg/CMakeLists.txt index 0ec275d9eec87ef6a7e92759763dae48aeb79741..642c1a3d252a724473b04f4435046daa449fa038 100644 --- a/cpp/src/core/test/test_nsg/CMakeLists.txt +++ b/cpp/src/core/test/test_nsg/CMakeLists.txt @@ -2,7 +2,6 @@ include_directories(/usr/local/include/gperftools) link_directories(/usr/local/lib) -#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free") add_definitions(-std=c++11 -O3 -lboost -march=native -Wall -DINFO) find_package(OpenMP) @@ -14,37 +13,29 @@ else () endif () message(${OpenMP_CXX_FLAGS}) -include_directories(${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg) -#include_directories(/home/zilliz/opt/app/pyenv/versions/3.6.8/include/python3.6m) -#include_directories(/home/zilliz/opt/app/pyenv/versions/3.6.8/envs/megasearch_testframework_dev/lib/python3.6/site-packages/numpy/core/include) +include_directories(${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg) -aux_source_directory(${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg nsg_src) - -#add_library(nsg_raw SHARED ${nsg_src}) -#target_link_libraries(nsg_raw -# gomp) +aux_source_directory(${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg nsg_src) set(interface_src - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg_index.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/common/exception.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/common/timer.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg_index.cpp + ${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp + ${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp + ${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp ../utils.cpp ) -add_executable(test_nsg - test_nsg.cpp - ${interface_src} - ${nsg_src} - ) -#target_link_libraries(test_nsg -# # libprofiler.so -## -ltcmalloc -## gomp -# ) +if(NOT TARGET test_nsg) + add_executable(test_nsg + test_nsg.cpp + ${interface_src} + ${nsg_src} + ) +endif() + target_link_libraries(test_nsg ${depend_libs} ${unittest_libs} ${basic_libs}) ############################## \ No newline at end of file diff --git a/cpp/unittest/CMakeLists.txt b/cpp/unittest/CMakeLists.txt index ac666c86a999b45ad2d453af0d021e9ba7923363..d251dce01a1fdf6e77d53269ecf08c6f7c53268a 100644 --- a/cpp/unittest/CMakeLists.txt +++ b/cpp/unittest/CMakeLists.txt @@ -38,9 +38,13 @@ set(unittest_libs cudart ) +foreach(dir ${CORE_INCLUDE_DIRS}) + include_directories(${dir}) +endforeach() + add_subdirectory(server) add_subdirectory(db) add_subdirectory(knowhere) add_subdirectory(metrics) -#add_subdirectory(scheduler) -#add_subdirectory(storage) \ No newline at end of file +add_subdirectory(scheduler) +add_subdirectory(storage) \ No newline at end of file diff --git a/cpp/unittest/db/CMakeLists.txt b/cpp/unittest/db/CMakeLists.txt index 64f56996d4624c8e3e9ca142878d7e2be76afecb..eae0b9bb833de9a98ad87f090d6115444ad30ef3 100644 --- a/cpp/unittest/db/CMakeLists.txt +++ b/cpp/unittest/db/CMakeLists.txt @@ -35,8 +35,6 @@ link_directories("/usr/local/cuda/lib64") include_directories(/usr/include/mysql) -#add_definitions(-DBOOST_ERROR_CODE_HEADER_ONLY) - set(db_test_src ${config_files} ${cache_srcs} @@ -69,25 +67,10 @@ set(db_libs set(knowhere_libs knowhere - SPTAGLibStatic - arrow - jemalloc_pic - faiss - openblas - lapack - tbb cudart cublas ) -if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON") - set(db_libs ${db_libs} ${MKL_LIBS} ${MKL_LIBS}) -else() - set(db_libs ${db_libs} - lapack - openblas) -endif() - target_link_libraries(db_test ${db_libs} ${knowhere_libs} ${unittest_libs}) install(TARGETS db_test DESTINATION unittest) diff --git a/cpp/unittest/knowhere/CMakeLists.txt b/cpp/unittest/knowhere/CMakeLists.txt index 6dc99d858f6c97ced9349f882dac7f3f9be7c6aa..089ff992fa665ba6c84b3f002a2ee22da4cd8ab9 100644 --- a/cpp/unittest/knowhere/CMakeLists.txt +++ b/cpp/unittest/knowhere/CMakeLists.txt @@ -8,13 +8,6 @@ set(helper set(knowhere_libs knowhere - SPTAGLibStatic - arrow - jemalloc_pic - faiss - openblas - lapack - tbb cudart cublas ) diff --git a/cpp/unittest/metrics/CMakeLists.txt b/cpp/unittest/metrics/CMakeLists.txt index 69f0f9e94a71d2ef98f155b3288aa142f367b8a3..1f0f5bd023465fa705e35810d44a0429fb9386f0 100644 --- a/cpp/unittest/metrics/CMakeLists.txt +++ b/cpp/unittest/metrics/CMakeLists.txt @@ -1,13 +1,3 @@ - - -#set(CURL_LIBRARY "-lcurl") -#find_package(CURL REQUIRED) -#INCLUDE_DIRECTORIES(${CURL_INCLUDE_DIR}) -# -#INCLUDE_DIRECTORIES(~/development/lib/usr/local/include) -#LINK_DIRECTORIES(~/development/lib/usr/local/lib) - - include_directories(../../src) @@ -72,13 +62,6 @@ add_executable(metrics_test ${count_test_src} ${require_files} ) set(knowhere_libs knowhere - SPTAGLibStatic - arrow - jemalloc_pic - faiss - openblas - lapack - tbb ) target_link_libraries(metrics_test @@ -96,12 +79,5 @@ target_link_libraries(metrics_test mysqlpp ${unittest_libs} ) -if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON") - target_link_libraries(metrics_test ${MKL_LIBS} ${MKL_LIBS}) -else() - target_link_libraries(metrics_test - lapack - openblas) -endif() install(TARGETS metrics_test DESTINATION unittest) \ No newline at end of file diff --git a/cpp/unittest/scheduler/CMakeLists.txt b/cpp/unittest/scheduler/CMakeLists.txt index 4f8b944611413a04cc49823a5e93b9dd7619e5c6..feb7d6178f75d9f31102360ea15090c228737817 100644 --- a/cpp/unittest/scheduler/CMakeLists.txt +++ b/cpp/unittest/scheduler/CMakeLists.txt @@ -33,8 +33,6 @@ link_directories("/usr/local/cuda/lib64") include_directories(/usr/include/mysql) -#add_definitions(-DBOOST_ERROR_CODE_HEADER_ONLY) - set(scheduler_test_src ${unittest_srcs} ${test_srcs} @@ -69,25 +67,10 @@ set(scheduler_libs set(knowhere_libs knowhere - SPTAGLibStatic - arrow - jemalloc_pic - faiss - openblas - lapack - tbb cudart cublas ) -if (${BUILD_FAISS_WITH_MKL} STREQUAL "ON") - set(db_libs ${db_libs} ${MKL_LIBS} ${MKL_LIBS}) -else () - set(db_libs ${db_libs} - lapack - openblas) -endif () - target_link_libraries(scheduler_test ${scheduler_libs} ${knowhere_libs} ${unittest_libs}) diff --git a/cpp/unittest/server/CMakeLists.txt b/cpp/unittest/server/CMakeLists.txt index cf7e62cb5eb8554077a7077229e9aa8198fa1e74..3dfca579940482c208d5289ad4db3e335d2f110a 100644 --- a/cpp/unittest/server/CMakeLists.txt +++ b/cpp/unittest/server/CMakeLists.txt @@ -34,9 +34,6 @@ cuda_add_executable(server_test set(require_libs knowhere - faiss - openblas - lapack stdc++ cudart cublas @@ -51,14 +48,6 @@ set(require_libs pthread ) -if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON") - set(require_libs ${require_libs} ${MKL_LIBS} ${MKL_LIBS}) -else() - set(require_libs ${require_libs} - lapack - openblas) -endif() - target_link_libraries(server_test ${require_libs} ${cuda_library}