提交 ec2bf996 编写于 作者: S starlord

Merge remote-tracking branch 'source/branch-0.4.0' into branch-0.4.0


Former-commit-id: 7186924615af54c6b5f478c32cb839ace35b607a
......@@ -28,6 +28,7 @@ Please mark all change in change log and use the ticket from JIRA.
- MS-471 - code coverage run failed
- MS-492 - Drop index failed if index have been created with index_type: FLAT
- MS-493 - Knowhere unittest crash
- MS-453 - GPU search error when nprobe set more than 1024
## Improvement
- MS-327 - Clean code for milvus
......@@ -90,6 +91,7 @@ Please mark all change in change log and use the ticket from JIRA.
- MS-482 - Change search stream transport to unary in grpc
- MS-487 - Define metric type in CreateTable
- MS-488 - Improve code format in scheduler
- MS-495 - cmake: integrated knowhere
## New Feature
- MS-343 - Implement ResourceMgr
......
......@@ -31,6 +31,7 @@ if(CMAKE_BUILD_TYPE STREQUAL "Release")
set(BUILD_TYPE "release")
else()
set(BUILD_TYPE "debug")
SET(CMAKE_VERBOSE_MAKEFILE on)
endif()
message(STATUS "Build type = ${BUILD_TYPE}")
......@@ -70,21 +71,21 @@ set(MILVUS_BINARY_DIR ${PROJECT_BINARY_DIR})
find_package(CUDA)
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES -arch sm_60 --expt-extended-lambda")
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g")
message("CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR}")
message("CUDA_NVCC_FLAGS=${CUDA_NVCC_FLAGS}")
message(STATUS "CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR}")
message(STATUS "CUDA_NVCC_FLAGS=${CUDA_NVCC_FLAGS}")
set(CMAKE_CXX_STANDARD 14)
set(CMAKE_CXX_STANDARD_REQUIRED on)
if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)")
message("building milvus_engine on x86 architecture")
message(STATUS "building milvus_engine on x86 architecture")
set(MILVUS_BUILD_ARCH x86_64)
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "(ppc)")
message("building milvus_engine on ppc architecture")
message(STATUS "building milvus_engine on ppc architecture")
set(MILVUS_BUILD_ARCH ppc64le)
else()
message("unknown processor type")
message("CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}")
message(WARNING "unknown processor type")
message(WARNING "CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}")
set(MILVUS_BUILD_ARCH unknown)
endif()
......@@ -94,6 +95,8 @@ else()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -DELPP_THREAD_SAFE -fopenmp")
endif()
set(ALLOW_DUPLICATE_CUSTOM_TARGETS TRUE)
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
include(ExternalProject)
......@@ -104,14 +107,6 @@ include(ThirdPartyPackages)
include_directories(${MILVUS_SOURCE_DIR})
link_directories(${MILVUS_BINARY_DIR})
if (NOT DEFINED KNOWHERE_BUILD_DIR)
message(FATAL_ERROR "You must set environment variable KNOWHERE_BUILD_DIR")
endif()
message(STATUS "Build with ${KNOWHERE_BUILD_DIR}")
include_directories(${KNOWHERE_BUILD_DIR}/include)
include_directories(${KNOWHERE_BUILD_DIR}/include/SPTAG/AnnService)
link_directories(${KNOWHERE_BUILD_DIR}/lib)
## Following should be check
set(MILVUS_ENGINE_INCLUDE ${PROJECT_SOURCE_DIR}/include)
......@@ -119,7 +114,7 @@ set(MILVUS_ENGINE_SRC ${PROJECT_SOURCE_DIR}/src)
add_compile_definitions(PROFILER=${PROFILER})
message("MILVUS_ENABLE_PROFILING = ${MILVUS_ENABLE_PROFILING}")
message(STATUS "MILVUS_ENABLE_PROFILING = ${MILVUS_ENABLE_PROFILING}")
if (MILVUS_ENABLE_PROFILING STREQUAL "ON")
ADD_DEFINITIONS(-DMILVUS_ENABLE_PROFILING)
endif()
......
......@@ -83,10 +83,6 @@ if [[ ! -d cmake_build ]]; then
MAKE_CLEAN="ON"
fi
pushd `pwd`/src/core
./build.sh ${KNOWHERE_OPTIONS}
popd
cd cmake_build
CUDA_COMPILER=/usr/local/cuda/bin/nvcc
......
......@@ -4,6 +4,13 @@
# Proprietary and confidential.
#-------------------------------------------------------------------------------
add_subdirectory(core)
set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE)
foreach(dir ${CORE_INCLUDE_DIRS})
include_directories(${dir})
endforeach()
aux_source_directory(cache cache_files)
aux_source_directory(config config_files)
aux_source_directory(server server_files)
......@@ -87,12 +94,6 @@ set(client_grpc_lib
set(third_party_libs
knowhere
SPTAGLibStatic
arrow
jemalloc_pic
faiss
openblas
lapack
easyloggingpp
sqlite
${client_grpc_lib}
......@@ -115,15 +116,6 @@ set(third_party_libs
cudart
)
if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON")
set(third_party_libs ${third_party_libs}
${MKL_LIBS})
else()
set(third_party_libs ${third_party_libs}
lapack
openblas)
endif()
if (MILVUS_ENABLE_PROFILING STREQUAL "ON")
set(third_party_libs ${third_party_libs}
gperftools
......@@ -171,13 +163,6 @@ set(server_libs
set(knowhere_libs
knowhere
SPTAGLibStatic
arrow
jemalloc_pic
faiss
openblas
lapack
tbb
)
add_executable(milvus_server
......@@ -195,8 +180,8 @@ target_link_libraries(milvus_server ${server_libs} ${knowhere_libs} ${third_part
install(TARGETS milvus_server DESTINATION bin)
install(FILES
${KNOWHERE_BUILD_DIR}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}tbb${CMAKE_SHARED_LIBRARY_SUFFIX}
${KNOWHERE_BUILD_DIR}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}tbb${CMAKE_SHARED_LIBRARY_SUFFIX}.2
${CMAKE_SOURCE_DIR}/src/core/thirdparty/tbb/${CMAKE_SHARED_LIBRARY_PREFIX}tbb${CMAKE_SHARED_LIBRARY_SUFFIX}
${CMAKE_SOURCE_DIR}/src/core/thirdparty/tbb/${CMAKE_SHARED_LIBRARY_PREFIX}tbb${CMAKE_SHARED_LIBRARY_SUFFIX}.2
${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}
${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3
${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3.2.4
......
......@@ -5,6 +5,7 @@
#-------------------------------------------------------------------------------
cmake_minimum_required(VERSION 3.14)
message(STATUS "---------------core--------------")
message(STATUS "Building using CMake version: ${CMAKE_VERSION}")
set(KNOWHERE_VERSION "0.1.0")
......@@ -38,18 +39,16 @@ endif()
MESSAGE(STATUS "CMAKE_CXX_FLAGS" ${CMAKE_CXX_FLAGS})
find_package(CUDA)
#set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES --expt-extended-lambda")
#set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g")
if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)")
message("building milvus_engine on x86 architecture")
message(STATUS "building milvus_engine on x86 architecture")
set(KNOWHERE_BUILD_ARCH x86_64)
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "(ppc)")
message("building milvus_engine on ppc architecture")
message(STATUS "building milvus_engine on ppc architecture")
set(KNOWHERE_BUILD_ARCH ppc64le)
else()
message("unknown processor type")
message("CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}")
message(WARNING "unknown processor type")
message(WARNING "CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}")
set(KNOWHERE_BUILD_ARCH unknown)
endif()
......@@ -60,14 +59,21 @@ else()
endif()
message(STATUS "Build type = ${BUILD_TYPE}")
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
set(CORE_SOURCE_DIR ${PROJECT_SOURCE_DIR})
set(CORE_BINARY_DIR ${PROJECT_BINARY_DIR})
message(STATUS "Core source dir: ${PROJECT_SOURCE_DIR}")
message(STATUS "Core binary dir: ${PROJECT_BINARY_DIR}")
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CORE_SOURCE_DIR}/cmake")
include(ExternalProject)
include(DefineOptions)
include(BuildUtils)
include(ThirdPartyPackages)
include(DefineOptionsCore)
include(BuildUtilsCore)
include(ThirdPartyPackagesCore)
add_subdirectory(src)
set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE)
if(BUILD_UNIT_TEST STREQUAL "ON")
add_subdirectory(test)
endif()
......
......@@ -75,7 +75,7 @@ function(ExternalProject_Use_Cache project_name package_file install_path)
"Extracting ${package_file} to ${install_path}"
COMMAND
${CMAKE_COMMAND} -E tar xzvf ${package_file} ${install_path}
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
WORKING_DIRECTORY ${CORE_BINARY_DIR}
)
ExternalProject_Add_StepTargets(${project_name} extract)
......
......@@ -104,7 +104,7 @@ macro(config_summary)
message(STATUS " Source directory: ${CMAKE_CURRENT_SOURCE_DIR}")
if(${CMAKE_EXPORT_COMPILE_COMMANDS})
message(
STATUS " Compile commands: ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json")
STATUS " Compile commands: ${CORE_BINARY_DIR}/compile_commands.json")
endif()
foreach(category ${KNOWHERE_OPTION_CATEGORIES})
......
......@@ -6,7 +6,6 @@
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
......@@ -18,7 +17,6 @@
set(KNOWHERE_THIRDPARTY_DEPENDENCIES
ARROW
BOOST
FAISS
GTest
LAPACK
......@@ -53,11 +51,8 @@ endmacro()
macro(resolve_dependency DEPENDENCY_NAME)
if (${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO")
#message(STATUS "Finding ${DEPENDENCY_NAME} package")
# find_package(${DEPENDENCY_NAME} QUIET)
# if (NOT ${DEPENDENCY_NAME}_FOUND)
#message(STATUS "${DEPENDENCY_NAME} package not found")
build_dependency(${DEPENDENCY_NAME})
# endif ()
elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "BUNDLED")
build_dependency(${DEPENDENCY_NAME})
elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "SYSTEM")
......@@ -120,7 +115,7 @@ endif (UNIX)
# ----------------------------------------------------------------------
# thirdparty directory
set(THIRDPARTY_DIR "${CMAKE_SOURCE_DIR}/thirdparty")
set(THIRDPARTY_DIR "${CORE_SOURCE_DIR}/thirdparty")
# ----------------------------------------------------------------------
# JFrog
......@@ -202,14 +197,6 @@ endif()
set(MAKE_BUILD_ARGS "-j8")
## Using make -j in sub-make is fragile
## see discussion https://github.com/apache/KNOWHERE/pull/2779
#if(${CMAKE_GENERATOR} MATCHES "Makefiles")
# set(MAKE_BUILD_ARGS "")
#else()
# # limit the maximum number of jobs for ninja
# set(MAKE_BUILD_ARGS "-j4")
#endif()
# ----------------------------------------------------------------------
# Find pthreads
......@@ -243,22 +230,11 @@ foreach(_VERSION_ENTRY ${TOOLCHAIN_VERSIONS_TXT})
set(${_LIB_NAME} "${_LIB_VERSION}")
endforeach()
if(DEFINED ENV{KNOWHERE_BOOST_URL})
set(BOOST_SOURCE_URL "$ENV{KNOWHERE_BOOST_URL}")
else()
string(REPLACE "." "_" BOOST_VERSION_UNDERSCORES ${BOOST_VERSION})
set(BOOST_SOURCE_URL
"https://dl.bintray.com/boostorg/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDERSCORES}.tar.gz"
)
endif()
if(DEFINED ENV{KNOWHERE_FAISS_URL})
set(FAISS_SOURCE_URL "$ENV{KNOWHERE_FAISS_URL}")
else()
set(FAISS_SOURCE_URL "http://192.168.1.105:6060/jinhai/faiss/-/archive/${FAISS_VERSION}/faiss-${FAISS_VERSION}.tar.gz")
# set(FAISS_SOURCE_URL "https://github.com/facebookresearch/faiss/archive/${FAISS_VERSION}.tar.gz")
# set(FAISS_SOURCE_URL "${CMAKE_SOURCE_DIR}/thirdparty/faiss-1.5.3")
message(STATUS ${FAISS_SOURCE_URL})
message(STATUS "FAISS URL = ${FAISS_SOURCE_URL}")
endif()
# set(FAISS_MD5 "a589663865a8558205533c8ac414278c")
# set(FAISS_MD5 "57da9c4f599cc8fa4260488b1c96e1cc") # commit-id 6dbdf75987c34a2c853bd172ea0d384feea8358c
......@@ -297,28 +273,18 @@ set(OPENBLAS_MD5 "8a110a25b819a4b94e8a9580702b6495")
# ----------------------------------------------------------------------
# ARROW
set(ARROW_PREFIX "${CORE_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep/cpp")
macro(build_arrow)
message(STATUS "Building Apache ARROW-${ARROW_VERSION} from source")
set(ARROW_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep/cpp")
set(ARROW_STATIC_LIB_NAME arrow)
# set(PARQUET_STATIC_LIB_NAME parquet)
# set(ARROW_CUDA_STATIC_LIB_NAME arrow_cuda)
set(ARROW_STATIC_LIB
set(ARROW_STATIC_LIB
"${ARROW_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${ARROW_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}"
)
# set(PARQUET_STATIC_LIB
# "${ARROW_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${PARQUET_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}"
# )
# set(ARROW_CUDA_STATIC_LIB
# "${ARROW_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${ARROW_CUDA_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}"
# )
set(ARROW_INCLUDE_DIR "${ARROW_PREFIX}/include")
set(ARROW_INCLUDE_DIR "${ARROW_PREFIX}/include")
set(ARROW_CMAKE_ARGS
${EP_COMMON_CMAKE_ARGS}
# "-DARROW_THRIFT_URL=${THRIFT_SOURCE_URL}"
#"env ARROW_THRIFT_URL=${THRIFT_SOURCE_URL}"
-DARROW_BUILD_STATIC=ON
-DARROW_BUILD_SHARED=OFF
-DARROW_PARQUET=OFF
......@@ -328,8 +294,7 @@ macro(build_arrow)
-DCMAKE_BUILD_TYPE=Release
-DARROW_DEPENDENCY_SOURCE=BUNDLED) #Build all arrow dependencies from source instead of calling find_package first
# set($ENV{ARROW_THRIFT_URL} ${THRIFT_SOURCE_URL})
if(USE_JFROG_CACHE STREQUAL "ON")
execute_process(COMMAND sh -c "git ls-remote --heads --tags ${ARROW_SOURCE_URL} ${ARROW_VERSION} | cut -f 1" OUTPUT_VARIABLE ARROW_LAST_COMMIT_ID)
if(${ARROW_LAST_COMMIT_ID} MATCHES "^[^#][a-z0-9]+")
......@@ -362,13 +327,13 @@ macro(build_arrow)
"${ARROW_STATIC_LIB}"
)
ExternalProject_Create_Cache(arrow_ep ${ARROW_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${ARROW_CACHE_URL})
ExternalProject_Create_Cache(arrow_ep ${ARROW_CACHE_PACKAGE_PATH} "${CORE_BINARY_DIR}/arrow_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${ARROW_CACHE_URL})
else()
file(DOWNLOAD ${ARROW_CACHE_URL} ${ARROW_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${ARROW_CACHE_URL} TO ${ARROW_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(arrow_ep ${ARROW_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
ExternalProject_Use_Cache(arrow_ep ${ARROW_CACHE_PACKAGE_PATH} ${CORE_BINARY_DIR})
endif()
endif()
else()
......@@ -402,10 +367,9 @@ macro(build_arrow)
set_target_properties(arrow
PROPERTIES IMPORTED_LOCATION "${ARROW_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${ARROW_INCLUDE_DIR}")
# INTERFACE_LINK_LIBRARIES thrift)
add_dependencies(arrow arrow_ep)
add_dependencies(arrow arrow_ep)
set(JEMALLOC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep-build/jemalloc_ep-prefix/src/jemalloc_ep")
set(JEMALLOC_PREFIX "${CORE_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep-build/jemalloc_ep-prefix/src/jemalloc_ep")
add_custom_command(TARGET arrow_ep POST_BUILD
COMMAND ${CMAKE_COMMAND} -E make_directory ${ARROW_PREFIX}/lib/
......@@ -414,7 +378,7 @@ macro(build_arrow)
endmacro()
if(KNOWHERE_WITH_ARROW)
if(KNOWHERE_WITH_ARROW AND NOT TARGET arrow_ep)
resolve_dependency(ARROW)
......@@ -422,191 +386,12 @@ if(KNOWHERE_WITH_ARROW)
include_directories(SYSTEM ${ARROW_INCLUDE_DIR})
endif()
# ----------------------------------------------------------------------
# Add Boost dependencies (code adapted from Apache Kudu (incubating))
set(Boost_USE_MULTITHREADED ON)
if(MSVC AND KNOWHERE_USE_STATIC_CRT)
set(Boost_USE_STATIC_RUNTIME ON)
endif()
set(Boost_ADDITIONAL_VERSIONS
"1.70.0"
"1.70"
"1.69.0"
"1.69"
"1.68.0"
"1.68"
"1.67.0"
"1.67"
"1.66.0"
"1.66"
"1.65.0"
"1.65"
"1.64.0"
"1.64"
"1.63.0"
"1.63"
"1.62.0"
"1.61"
"1.61.0"
"1.62"
"1.60.0"
"1.60")
# TODO
if(KNOWHERE_BOOST_VENDORED)
# system thread serialization wserialization regex
set(BOOST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/boost_ep-prefix/src/boost_ep")
set(BOOST_LIB_DIR "${BOOST_PREFIX}/stage/lib")
set(BOOST_BUILD_LINK "static")
set(BOOST_STATIC_SYSTEM_LIBRARY
"${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_system${CMAKE_STATIC_LIBRARY_SUFFIX}"
)
set(BOOST_STATIC_FILESYSTEM_LIBRARY
"${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_filesystem${CMAKE_STATIC_LIBRARY_SUFFIX}"
)
set(BOOST_STATIC_SERIALIZATION_LIBRARY
"${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_serialization${CMAKE_STATIC_LIBRARY_SUFFIX}"
)
set(BOOST_STATIC_WSERIALIZATION_LIBRARY
"${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_wserialization${CMAKE_STATIC_LIBRARY_SUFFIX}"
)
set(BOOST_STATIC_REGEX_LIBRARY
"${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_regex${CMAKE_STATIC_LIBRARY_SUFFIX}"
)
set(BOOST_STATIC_THREAD_LIBRARY
"${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_thread${CMAKE_STATIC_LIBRARY_SUFFIX}"
)
set(BOOST_SYSTEM_LIBRARY boost_system_static)
set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_static)
set(BOOST_SERIALIZATION_LIBRARY boost_serialization_static)
set(BOOST_WSERIALIZATION_LIBRARY boost_wserialization_static)
set(BOOST_REGEX_LIBRARY boost_regex_static)
set(BOOST_THREAD_LIBRARY boost_thread_static)
if(KNOWHERE_BOOST_HEADER_ONLY)
set(BOOST_BUILD_PRODUCTS)
set(BOOST_CONFIGURE_COMMAND "")
set(BOOST_BUILD_COMMAND "")
else()
set(BOOST_BUILD_PRODUCTS ${BOOST_STATIC_SYSTEM_LIBRARY}
${BOOST_STATIC_FILESYSTEM_LIBRARY} ${BOOST_STATIC_SERIALIZATION_LIBRARY}
${BOOST_STATIC_WSERIALIZATION_LIBRARY} ${BOOST_STATIC_REGEX_LIBRARY}
${BOOST_STATIC_THREAD_LIBRARY})
set(BOOST_CONFIGURE_COMMAND "./bootstrap.sh" "--prefix=${BOOST_PREFIX}"
"--with-libraries=filesystem,serialization,wserialization,system,thread,regex")
if("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
set(BOOST_BUILD_VARIANT "debug")
else()
set(BOOST_BUILD_VARIANT "release")
endif()
set(BOOST_BUILD_COMMAND
"./b2"
"link=${BOOST_BUILD_LINK}"
"variant=${BOOST_BUILD_VARIANT}"
"cxxflags=-fPIC")
add_thirdparty_lib(boost_system STATIC_LIB "${BOOST_STATIC_SYSTEM_LIBRARY}")
add_thirdparty_lib(boost_filesystem STATIC_LIB "${BOOST_STATIC_FILESYSTEM_LIBRARY}")
add_thirdparty_lib(boost_serialization STATIC_LIB "${BOOST_STATIC_SERIALIZATION_LIBRARY}")
add_thirdparty_lib(boost_wserialization STATIC_LIB "${BOOST_STATIC_WSERIALIZATION_LIBRARY}")
add_thirdparty_lib(boost_regex STATIC_LIB "${BOOST_STATIC_REGEX_LIBRARY}")
add_thirdparty_lib(boost_thread STATIC_LIB "${BOOST_STATIC_THREAD_LIBRARY}")
set(KNOWHERE_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY} ${BOOST_SERIALIZATION_LIBRARY}
${BOOST_WSERIALIZATION_LIBRARY} ${BOOST_REGEX_LIBRARY} ${BOOST_THREAD_LIBRARY})
endif()
externalproject_add(boost_ep
URL
${BOOST_SOURCE_URL}
BUILD_BYPRODUCTS
${BOOST_BUILD_PRODUCTS}
BUILD_IN_SOURCE
1
CONFIGURE_COMMAND
${BOOST_CONFIGURE_COMMAND}
BUILD_COMMAND
${BOOST_BUILD_COMMAND}
INSTALL_COMMAND
""
${EP_LOG_OPTIONS})
set(Boost_INCLUDE_DIR "${BOOST_PREFIX}")
set(Boost_INCLUDE_DIRS "${BOOST_INCLUDE_DIR}")
add_dependencies(boost_system_static boost_ep)
add_dependencies(boost_filesystem_static boost_ep)
add_dependencies(boost_serialization_static boost_ep)
add_dependencies(boost_wserialization_static boost_ep)
add_dependencies(boost_regex_static boost_ep)
add_dependencies(boost_thread_static boost_ep)
#else()
# if(MSVC)
# # disable autolinking in boost
# add_definitions(-DBOOST_ALL_NO_LIB)
# endif()
# if(DEFINED ENV{BOOST_ROOT} OR DEFINED BOOST_ROOT)
# # In older versions of CMake (such as 3.2), the system paths for Boost will
# # In older versions of CMake (such as 3.2), the system paths for Boost will
# # be looked in first even if we set $BOOST_ROOT or pass -DBOOST_ROOT
# set(Boost_NO_SYSTEM_PATHS ON)
# endif()
# if(KNOWHERE_BOOST_USE_SHARED)
# # Find shared Boost libraries.
# set(Boost_USE_STATIC_LIBS OFF)
# set(BUILD_SHARED_LIBS_KEEP ${BUILD_SHARED_LIBS})
# set(BUILD_SHARED_LIBS ON)
#
# if(MSVC)
# # force all boost libraries to dynamic link
# add_definitions(-DBOOST_ALL_DYN_LINK)
# endif()
#
# if(KNOWHERE_BOOST_HEADER_ONLY)
# find_package(Boost REQUIRED)
# else()
# find_package(Boost COMPONENTS serialization system filesystem REQUIRED)
# set(BOOST_SYSTEM_LIBRARY Boost::system)
# set(BOOST_FILESYSTEM_LIBRARY Boost::filesystem)
# set(BOOST_SERIALIZATION_LIBRARY Boost::serialization)
# set(KNOWHERE_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY})
# endif()
# set(BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS_KEEP})
# unset(BUILD_SHARED_LIBS_KEEP)
# else()
# # Find static boost headers and libs
# # TODO Differentiate here between release and debug builds
# set(Boost_USE_STATIC_LIBS ON)
# if(KNOWHERE_BOOST_HEADER_ONLY)
# find_package(Boost REQUIRED)
# else()
# find_package(Boost COMPONENTS serialization system filesystem REQUIRED)
# set(BOOST_SYSTEM_LIBRARY Boost::system)
# set(BOOST_FILESYSTEM_LIBRARY Boost::filesystem)
# set(BOOST_SERIALIZATION_LIBRARY Boost::serialization)
# set(KNOWHERE_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY})
# endif()
# endif()
endif()
#message(STATUS "Boost include dir: " ${Boost_INCLUDE_DIR})
#message(STATUS "Boost libraries: " ${Boost_LIBRARIES})
include_directories(SYSTEM ${Boost_INCLUDE_DIR})
link_directories(SYSTEM ${BOOST_LIB_DIR})
# ----------------------------------------------------------------------
# OpenBLAS
macro(build_openblas)
message(STATUS "Building OpenBLAS-${OPENBLAS_VERSION} from source")
set(OPENBLAS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/openblas_ep-prefix/src/openblas_ep")
set(OPENBLAS_PREFIX "${CORE_BINARY_DIR}/openblas_ep-prefix/src/openblas_ep")
set(OPENBLAS_INCLUDE_DIR "${OPENBLAS_PREFIX}/include")
set(OPENBLAS_STATIC_LIB
"${OPENBLAS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}openblas${CMAKE_STATIC_LIBRARY_SUFFIX}")
......@@ -639,13 +424,13 @@ macro(build_openblas)
BUILD_BYPRODUCTS
${OPENBLAS_STATIC_LIB})
ExternalProject_Create_Cache(openblas_ep ${OPENBLAS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/openblas_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${OPENBLAS_CACHE_URL})
ExternalProject_Create_Cache(openblas_ep ${OPENBLAS_CACHE_PACKAGE_PATH} "${CORE_BINARY_DIR}/openblas_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${OPENBLAS_CACHE_URL})
else()
file(DOWNLOAD ${OPENBLAS_CACHE_URL} ${OPENBLAS_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${OPENBLAS_CACHE_URL} TO ${OPENBLAS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(openblas_ep ${OPENBLAS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
ExternalProject_Use_Cache(openblas_ep ${OPENBLAS_CACHE_PACKAGE_PATH} ${CORE_BINARY_DIR})
endif()
endif()
else()
......@@ -678,20 +463,12 @@ macro(build_openblas)
add_dependencies(openblas openblas_ep)
endmacro()
#if(KNOWHERE_WITH_OPENBLAS)
# resolve_dependency(OpenBLAS)
#
# get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES)
# include_directories(SYSTEM "${OPENBLAS_INCLUDE_DIR}")
# link_directories(SYSTEM ${OPENBLAS_PREFIX}/lib)
#endif()
# ----------------------------------------------------------------------
# LAPACK
macro(build_lapack)
message(STATUS "Building LAPACK-${LAPACK_VERSION} from source")
set(LAPACK_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/lapack_ep-prefix/src/lapack_ep")
set(LAPACK_PREFIX "${CORE_BINARY_DIR}/lapack_ep-prefix/src/lapack_ep")
set(LAPACK_INCLUDE_DIR "${LAPACK_PREFIX}/include")
set(LAPACK_STATIC_LIB
"${LAPACK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}lapack${CMAKE_STATIC_LIBRARY_SUFFIX}")
......@@ -723,13 +500,13 @@ macro(build_lapack)
BUILD_BYPRODUCTS
${LAPACK_STATIC_LIB})
ExternalProject_Create_Cache(lapack_ep ${LAPACK_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/lapack_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${LAPACK_CACHE_URL})
ExternalProject_Create_Cache(lapack_ep ${LAPACK_CACHE_PACKAGE_PATH} "${CORE_BINARY_DIR}/lapack_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${LAPACK_CACHE_URL})
else()
file(DOWNLOAD ${LAPACK_CACHE_URL} ${LAPACK_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${LAPACK_CACHE_URL} TO ${LAPACK_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(lapack_ep ${LAPACK_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
ExternalProject_Use_Cache(lapack_ep ${LAPACK_CACHE_PACKAGE_PATH} ${CORE_BINARY_DIR})
endif()
endif()
else()
......@@ -756,14 +533,6 @@ macro(build_lapack)
add_dependencies(lapack lapack_ep)
endmacro()
#if(KNOWHERE_WITH_LAPACK)
# resolve_dependency(LAPACK)
#
# get_target_property(LAPACK_INCLUDE_DIR lapack INTERFACE_INCLUDE_DIRECTORIES)
# include_directories(SYSTEM "${LAPACK_INCLUDE_DIR}")
# link_directories(SYSTEM ${LAPACK_PREFIX}/lib)
#endif()
# ----------------------------------------------------------------------
# Google gtest
......@@ -780,7 +549,7 @@ macro(build_gtest)
-Wno-ignored-attributes)
endif()
set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep")
set(GTEST_PREFIX "${CORE_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep")
set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include")
set(GTEST_STATIC_LIB
"${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}")
......@@ -822,13 +591,13 @@ macro(build_gtest)
${GTEST_CMAKE_ARGS}
${EP_LOG_OPTIONS})
ExternalProject_Create_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${GTEST_CACHE_URL})
ExternalProject_Create_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} "${CORE_BINARY_DIR}/googletest_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${GTEST_CACHE_URL})
else()
file(DOWNLOAD ${GTEST_CACHE_URL} ${GTEST_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${GTEST_CACHE_URL} TO ${GTEST_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
ExternalProject_Use_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} ${CORE_BINARY_DIR})
endif()
endif()
else()
......@@ -871,8 +640,7 @@ macro(build_gtest)
endmacro()
if (KNOWHERE_BUILD_TESTS)
#message(STATUS "Resolving gtest dependency")
if (KNOWHERE_BUILD_TESTS AND NOT TARGET googletest_ep)
resolve_dependency(GTest)
if(NOT GTEST_VENDORED)
......@@ -889,17 +657,12 @@ endif()
macro(build_faiss)
message(STATUS "Building FAISS-${FAISS_VERSION} from source")
set(FAISS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/faiss_ep-prefix/src/faiss_ep")
set(FAISS_PREFIX "${CORE_BINARY_DIR}/faiss_ep-prefix/src/faiss_ep")
set(FAISS_INCLUDE_DIR "${FAISS_PREFIX}/include")
set(FAISS_STATIC_LIB
"${FAISS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}faiss${CMAKE_STATIC_LIBRARY_SUFFIX}")
# add_custom_target(faiss_dependencies)
# add_dependencies(faiss_dependencies openblas_ep)
# add_dependencies(faiss_dependencies openblas)
# get_target_property(FAISS_OPENBLAS_LIB_DIR openblas IMPORTED_LOCATION)
# get_filename_component(FAISS_OPENBLAS_LIB "${FAISS_OPENBLAS_LIB_DIR}" DIRECTORY)
set(FAISS_CONFIGURE_ARGS
"--prefix=${FAISS_PREFIX}"
"CFLAGS=${EP_C_FLAGS}"
......@@ -907,13 +670,7 @@ macro(build_faiss)
"LDFLAGS=-L${OPENBLAS_PREFIX}/lib -L${LAPACK_PREFIX}/lib -lopenblas -llapack"
--without-python)
# if(OPENBLAS_STATIC_LIB)
# set(OPENBLAS_LIBRARY ${OPENBLAS_STATIC_LIB})
# else()
# set(OPENBLAS_LIBRARY ${OPENBLAS_SHARED_LIB})
# endif()
# set(FAISS_DEPENDENCIES ${FAISS_DEPENDENCIES} ${OPENBLAS_LIBRARY})
if(${KNOWHERE_WITH_FAISS_GPU_VERSION} STREQUAL "ON")
set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
"--with-cuda=${CUDA_TOOLKIT_ROOT_DIR}"
......@@ -924,9 +681,7 @@ macro(build_faiss)
endif()
if(USE_JFROG_CACHE STREQUAL "ON")
# Check_Last_Modify("${CMAKE_SOURCE_DIR}/thirdparty/faiss_cache_check_lists.txt" "${CMAKE_SOURCE_DIR}" FAISS_LAST_MODIFIED_COMMIT_ID)
string(MD5 FAISS_COMBINE_MD5 "${FAISS_MD5}${LAPACK_MD5}${OPENBLAS_MD5}")
# string(MD5 FAISS_COMBINE_MD5 "${FAISS_LAST_MODIFIED_COMMIT_ID}${LAPACK_MD5}${OPENBLAS_MD5}")
set(FAISS_CACHE_PACKAGE_NAME "faiss_${FAISS_COMBINE_MD5}.tar.gz")
set(FAISS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${FAISS_CACHE_PACKAGE_NAME}")
set(FAISS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${FAISS_CACHE_PACKAGE_NAME}")
......@@ -952,13 +707,13 @@ macro(build_faiss)
ExternalProject_Add_StepDependencies(faiss_ep build openblas_ep lapack_ep)
ExternalProject_Create_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/faiss_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${FAISS_CACHE_URL})
ExternalProject_Create_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} "${CORE_BINARY_DIR}/faiss_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${FAISS_CACHE_URL})
else()
file(DOWNLOAD ${FAISS_CACHE_URL} ${FAISS_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${FAISS_CACHE_URL} TO ${FAISS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
ExternalProject_Use_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} ${CORE_BINARY_DIR})
endif()
endif()
else()
......@@ -990,14 +745,10 @@ macro(build_faiss)
INTERFACE_LINK_LIBRARIES "openblas;lapack" )
add_dependencies(faiss faiss_ep)
#add_dependencies(faiss openblas_ep)
#add_dependencies(faiss lapack_ep)
#target_link_libraries(faiss ${OPENBLAS_PREFIX}/lib)
#target_link_libraries(faiss ${LAPACK_PREFIX}/lib)
endmacro()
if(KNOWHERE_WITH_FAISS)
if(KNOWHERE_WITH_FAISS AND NOT TARGET faiss_ep)
resolve_dependency(OpenBLAS)
get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES)
......@@ -1012,8 +763,5 @@ if(KNOWHERE_WITH_FAISS)
resolve_dependency(FAISS)
get_target_property(FAISS_INCLUDE_DIR faiss INTERFACE_INCLUDE_DIRECTORIES)
include_directories(SYSTEM "${FAISS_INCLUDE_DIR}")
# include_directories(SYSTEM "${CMAKE_CURRENT_BINARY_DIR}/faiss_ep-prefix/src/")
# link_directories(SYSTEM ${FAISS_PREFIX}/)
link_directories(SYSTEM ${FAISS_PREFIX}/lib/)
# link_directories(SYSTEM ${FAISS_PREFIX}/gpu/)
endif()
set(TBB_DIR ${CMAKE_SOURCE_DIR}/thirdparty/tbb)
set(TBB_DIR ${CORE_SOURCE_DIR}/thirdparty/tbb)
set(TBB_LIBRARIES ${TBB_DIR}/libtbb.so)
include_directories(${TBB_DIR}/include)
include_directories(${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
link_directories(${CUDA_TOOLKIT_ROOT_DIR}/lib64)
include_directories(${CMAKE_SOURCE_DIR}/include)
include_directories(${CMAKE_SOURCE_DIR}/thirdparty)
include_directories(${CMAKE_SOURCE_DIR}/thirdparty/SPTAG/AnnService)
include_directories(${CMAKE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include)
include_directories(${CORE_SOURCE_DIR}/include)
include_directories(${CORE_SOURCE_DIR}/thirdparty)
include_directories(${CORE_SOURCE_DIR}/thirdparty/SPTAG/AnnService)
include_directories(${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include)
set(SPTAG_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../thirdparty/SPTAG)
set(SPTAG_SOURCE_DIR ${CORE_SOURCE_DIR}/thirdparty/SPTAG)
file(GLOB HDR_FILES
${SPTAG_SOURCE_DIR}/AnnService/inc/Core/*.h
${SPTAG_SOURCE_DIR}/AnnService/inc/Core/Common/*.h
......@@ -24,9 +24,9 @@ file(GLOB SRC_FILES
${SPTAG_SOURCE_DIR}/AnnService/src/Core/KDT/*.cpp
${SPTAG_SOURCE_DIR}/AnnService/src/Helper/*.cpp)
#add_library(SPTAGLib SHARED ${SRC_FILES} ${HDR_FILES})
#target_link_libraries(SPTAGLib ${TBB_LIBRARIES})
add_library(SPTAGLibStatic STATIC ${SRC_FILES} ${HDR_FILES})
if(NOT TARGET SPTAGLibStatic)
add_library(SPTAGLibStatic STATIC ${SRC_FILES} ${HDR_FILES})
endif()
set(external_srcs
knowhere/adapter/sptag.cpp
......@@ -51,14 +51,13 @@ set(index_srcs
)
set(depend_libs
# libtcmalloc.a
SPTAGLibStatic
${TBB_LIBRARIES}
faiss
openblas
lapack
arrow
jemalloc_pic
${ARROW_PREFIX}/lib/libjemalloc_pic.a
cudart
cublas
gomp
......@@ -66,30 +65,19 @@ set(depend_libs
pthread
)
add_library(
knowhere STATIC
${external_srcs}
${index_srcs}
)
#target_compile_options(knowhere PUBLIC "-fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free")
if(NOT TARGET knowhere)
add_library(
knowhere STATIC
${external_srcs}
${index_srcs}
)
endif()
target_link_libraries(
knowhere
${depend_libs}
)
#add_library(
# knowhereS SHARED
# ${external_srcs}
# ${index_srcs}
#)
#
#target_link_libraries(
# knowhereS
## ${TBB_LIBRARIES}
# ${depend_libs}
#)
INSTALL(TARGETS
knowhere
SPTAGLibStatic
......@@ -98,7 +86,6 @@ INSTALL(TARGETS
INSTALL(FILES
${ARROW_STATIC_LIB}
# ${PARQUET_STATIC_LIB}
${ARROW_PREFIX}/lib/libjemalloc_pic.a
${FAISS_STATIC_LIB}
${LAPACK_STATIC_LIB}
......@@ -112,24 +99,35 @@ INSTALL(FILES ${OPENBLAS_REAL_STATIC_LIB}
DESTINATION lib
)
INSTALL(FILES ${CMAKE_SOURCE_DIR}/thirdparty/tbb/libtbb.so.2
# RENAME "libtbb.so.2"
INSTALL(FILES ${CORE_SOURCE_DIR}/thirdparty/tbb/libtbb.so.2
DESTINATION lib
)
INSTALL(FILES ${CMAKE_SOURCE_DIR}/thirdparty/tbb/libtbb.so
# RENAME "libtbb.so"
INSTALL(FILES ${CORE_SOURCE_DIR}/thirdparty/tbb/libtbb.so
DESTINATION lib
)
set(CORE_INCLUDE_DIRS
${CORE_SOURCE_DIR}/include
${CORE_SOURCE_DIR}/thirdparty
${CORE_SOURCE_DIR}/thirdparty/SPTAG/AnnService
${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include
${ARROW_INCLUDE_DIR}
${FAISS_INCLUDE_DIR}
${OPENBLAS_INCLUDE_DIR}
${LAPACK_INCLUDE_DIR}
${CORE_SOURCE_DIR}/thirdparty/tbb/include
)
set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE)
INSTALL(DIRECTORY
${CMAKE_SOURCE_DIR}/include/knowhere
${CMAKE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons
${CMAKE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons_ext
${CORE_SOURCE_DIR}/include/knowhere
${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons
${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons_ext
${ARROW_INCLUDE_DIR}/arrow
# ${ARROW_INCLUDE_DIR}/parquet
${FAISS_PREFIX}/include/faiss
${OPENBLAS_INCLUDE_DIR}/
${CMAKE_SOURCE_DIR}/thirdparty/tbb/include/tbb
${CORE_SOURCE_DIR}/thirdparty/tbb/include/tbb
DESTINATION
include)
......
include_directories(${CMAKE_SOURCE_DIR}/thirdparty)
include_directories(${CMAKE_SOURCE_DIR}/thirdparty/SPTAG/AnnService)
include_directories(${CMAKE_SOURCE_DIR}/include)
include_directories(${CMAKE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include)
include_directories(${CORE_SOURCE_DIR}/thirdparty)
include_directories(${CORE_SOURCE_DIR}/thirdparty/SPTAG/AnnService)
include_directories(${CORE_SOURCE_DIR}/include)
include_directories(${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include)
include_directories(/usr/local/cuda/include)
link_directories(/usr/local/cuda/lib64)
link_directories(${CMAKE_SOURCE_DIR}/thirdparty/tbb)
link_directories(${CORE_SOURCE_DIR}/thirdparty/tbb)
set(unittest_libs
gtest gmock gtest_main gmock_main)
message(STATUS "arrow prefix: ${ARROW_PREFIX}")
message(STATUS "libjemalloc_pic path: ${ARROW_PREFIX}/lib/libjemalloc_pic.a")
set(depend_libs
faiss openblas lapack
arrow jemalloc_pic
arrow "${ARROW_PREFIX}/lib/libjemalloc_pic.a"
tbb
)
......@@ -22,57 +24,55 @@ set(basic_libs
#<IVF-TEST>
#<<<<<<<<<<
set(ivf_srcs
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/common/exception.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/common/timer.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp
${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp
${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp
${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp
utils.cpp
)
add_executable(test_ivf test_ivf.cpp ${ivf_srcs})
if(NOT TARGET test_ivf)
add_executable(test_ivf test_ivf.cpp ${ivf_srcs})
endif()
target_link_libraries(test_ivf ${depend_libs} ${unittest_libs} ${basic_libs})
#<<<<<<<<<<
#<IDMAP-TEST>
#<<<<<<<<<<
set(idmap_srcs
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/common/exception.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/common/timer.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp
${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp
${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp
${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp
utils.cpp
)
add_executable(test_idmap test_idmap.cpp ${idmap_srcs})
if(NOT TARGET test_idmap)
add_executable(test_idmap test_idmap.cpp ${idmap_srcs})
endif()
target_link_libraries(test_idmap ${depend_libs} ${unittest_libs} ${basic_libs})
#<<<<<<<<<<
#<KDT-TEST>
#<<<<<<<<<<
set(kdt_srcs
${CMAKE_SOURCE_DIR}/src/knowhere/index/preprocessor/normalize.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/kdt_parameters.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/cpu_kdt_rng.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/adapter/sptag.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/common/exception.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/adapter/arrow.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/common/timer.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/preprocessor/normalize.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/kdt_parameters.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cpu_kdt_rng.cpp
${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp
${CORE_SOURCE_DIR}/src/knowhere/adapter/sptag.cpp
${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp
${CORE_SOURCE_DIR}/src/knowhere/adapter/arrow.cpp
${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp
utils.cpp
)
add_executable(test_kdt test_kdt.cpp ${kdt_srcs})
if(NOT TARGET test_kdt)
add_executable(test_kdt test_kdt.cpp ${kdt_srcs})
endif()
target_link_libraries(test_kdt
SPTAGLibStatic
${depend_libs} ${unittest_libs} ${basic_libs})
#<<<<<<<<<<
add_subdirectory(faiss_ori)
add_subdirectory(test_nsg)
......
include_directories(${CMAKE_SOURCE_DIR}/thirdparty)
include_directories(${CMAKE_SOURCE_DIR}/include)
include_directories(${CORE_SOURCE_DIR}/thirdparty)
include_directories(${CORE_SOURCE_DIR}/include)
include_directories(/usr/local/cuda/include)
link_directories(/usr/local/cuda/lib64)
link_directories(${CMAKE_SOURCE_DIR}/thirdparty/tbb)
link_directories(${CORE_SOURCE_DIR}/thirdparty/tbb)
set(unittest_libs
gtest gmock gtest_main gmock_main)
set(depend_libs
faiss openblas lapack
arrow jemalloc_pic
arrow ${ARROW_PREFIX}/lib/libjemalloc_pic.a
tbb
)
......@@ -20,7 +20,7 @@ set(basic_libs
#<GPU-TEST>
#<<<<<<<<<<
add_executable(test_gpu gpuresource_test.cpp)
if(NOT TARGET test_gpu)
add_executable(test_gpu gpuresource_test.cpp)
endif()
target_link_libraries(test_gpu ${depend_libs} ${unittest_libs} ${basic_libs})
#<<<<<<<<<<
......@@ -2,7 +2,6 @@
include_directories(/usr/local/include/gperftools)
link_directories(/usr/local/lib)
#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free")
add_definitions(-std=c++11 -O3 -lboost -march=native -Wall -DINFO)
find_package(OpenMP)
......@@ -14,37 +13,29 @@ else ()
endif ()
message(${OpenMP_CXX_FLAGS})
include_directories(${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg)
#include_directories(/home/zilliz/opt/app/pyenv/versions/3.6.8/include/python3.6m)
#include_directories(/home/zilliz/opt/app/pyenv/versions/3.6.8/envs/megasearch_testframework_dev/lib/python3.6/site-packages/numpy/core/include)
include_directories(${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg)
aux_source_directory(${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg nsg_src)
#add_library(nsg_raw SHARED ${nsg_src})
#target_link_libraries(nsg_raw
# gomp)
aux_source_directory(${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg nsg_src)
set(interface_src
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg_index.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/common/exception.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/common/timer.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg_index.cpp
${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp
${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp
${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp
../utils.cpp
)
add_executable(test_nsg
test_nsg.cpp
${interface_src}
${nsg_src}
)
#target_link_libraries(test_nsg
# # libprofiler.so
## -ltcmalloc
## gomp
# )
if(NOT TARGET test_nsg)
add_executable(test_nsg
test_nsg.cpp
${interface_src}
${nsg_src}
)
endif()
target_link_libraries(test_nsg ${depend_libs} ${unittest_libs} ${basic_libs})
##############################
\ No newline at end of file
......@@ -72,8 +72,11 @@ ErrorCode VecIndexImpl::Search(const long &nq, const float *xq, float *dist, lon
auto k = cfg["k"].as<int>();
auto dataset = GenDataset(nq, dim, xq);
Config search_cfg;
auto res = index_->Search(dataset, cfg);
Config search_cfg = cfg;
ParameterValidation(type, search_cfg);
auto res = index_->Search(dataset, search_cfg);
auto ids_array = res->array()[0];
auto dis_array = res->array()[1];
......
......@@ -71,7 +71,7 @@ size_t FileIOWriter::operator()(void *ptr, size_t size) {
}
VecIndexPtr GetVecIndexFactory(const IndexType &type, const Config& cfg) {
VecIndexPtr GetVecIndexFactory(const IndexType &type, const Config &cfg) {
std::shared_ptr<zilliz::knowhere::VectorIndex> index;
auto gpu_device = cfg.get_with_default("gpu_id", 0);
switch (type) {
......@@ -235,6 +235,31 @@ void AutoGenParams(const IndexType &type, const long &size, zilliz::knowhere::Co
}
}
#if CUDA_VERSION > 9000
#define GPU_MAX_NRPOBE 2048
#else
#define GPU_MAX_NRPOBE 1024
#endif
void ParameterValidation(const IndexType &type, Config &cfg) {
switch (type) {
case IndexType::FAISS_IVFSQ8_GPU:
case IndexType::FAISS_IVFFLAT_GPU:
case IndexType::FAISS_IVFPQ_GPU: {
if (cfg.get_with_default("nprobe", 0) != 0) {
auto nprobe = cfg["nprobe"].as<int>();
if (nprobe > GPU_MAX_NRPOBE) {
WRAPPER_LOG_WARNING << "When search with GPU, nprobe shoud be no more than " << GPU_MAX_NRPOBE << ", but you passed " << nprobe
<< ". Search with " << GPU_MAX_NRPOBE << " instead";
cfg.insert_or_assign("nprobe", GPU_MAX_NRPOBE);
}
}
break;
}
default:break;
}
}
IndexType ConvertToCpuIndexType(const IndexType &type) {
// TODO(linxj): add IDMAP
switch (type) {
......
......@@ -14,6 +14,8 @@
#include "knowhere/common/config.h"
#include "knowhere/common/binary_set.h"
#include "cuda.h"
namespace zilliz {
namespace milvus {
......@@ -90,6 +92,8 @@ extern VecIndexPtr LoadVecIndex(const IndexType &index_type, const zilliz::knowh
extern void AutoGenParams(const IndexType& type, const long& size, Config& cfg);
extern void ParameterValidation(const IndexType& type, Config& cfg);
extern IndexType ConvertToCpuIndexType(const IndexType& type);
extern IndexType ConvertToGpuIndexType(const IndexType& type);
......
......@@ -38,6 +38,10 @@ set(unittest_libs
cudart
)
foreach(dir ${CORE_INCLUDE_DIRS})
include_directories(${dir})
endforeach()
add_subdirectory(server)
add_subdirectory(db)
add_subdirectory(knowhere)
......
......@@ -35,8 +35,6 @@ link_directories("/usr/local/cuda/lib64")
include_directories(/usr/include/mysql)
#add_definitions(-DBOOST_ERROR_CODE_HEADER_ONLY)
set(db_test_src
${config_files}
${cache_srcs}
......@@ -69,25 +67,10 @@ set(db_libs
set(knowhere_libs
knowhere
SPTAGLibStatic
arrow
jemalloc_pic
faiss
openblas
lapack
tbb
cudart
cublas
)
if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON")
set(db_libs ${db_libs} ${MKL_LIBS} ${MKL_LIBS})
else()
set(db_libs ${db_libs}
lapack
openblas)
endif()
target_link_libraries(db_test ${db_libs} ${knowhere_libs} ${unittest_libs})
install(TARGETS db_test DESTINATION unittest)
......
......@@ -8,13 +8,6 @@ set(helper
set(knowhere_libs
knowhere
SPTAGLibStatic
arrow
jemalloc_pic
faiss
openblas
lapack
tbb
cudart
cublas
)
......
#set(CURL_LIBRARY "-lcurl")
#find_package(CURL REQUIRED)
#INCLUDE_DIRECTORIES(${CURL_INCLUDE_DIR})
#
#INCLUDE_DIRECTORIES(~/development/lib/usr/local/include)
#LINK_DIRECTORIES(~/development/lib/usr/local/lib)
include_directories(../../src)
......@@ -72,13 +62,6 @@ add_executable(metrics_test ${count_test_src} ${require_files} )
set(knowhere_libs
knowhere
SPTAGLibStatic
arrow
jemalloc_pic
faiss
openblas
lapack
tbb
)
target_link_libraries(metrics_test
......@@ -96,12 +79,5 @@ target_link_libraries(metrics_test
mysqlpp
${unittest_libs}
)
if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON")
target_link_libraries(metrics_test ${MKL_LIBS} ${MKL_LIBS})
else()
target_link_libraries(metrics_test
lapack
openblas)
endif()
install(TARGETS metrics_test DESTINATION unittest)
\ No newline at end of file
......@@ -33,8 +33,6 @@ link_directories("/usr/local/cuda/lib64")
include_directories(/usr/include/mysql)
#add_definitions(-DBOOST_ERROR_CODE_HEADER_ONLY)
set(scheduler_test_src
${unittest_srcs}
${test_srcs}
......@@ -69,25 +67,10 @@ set(scheduler_libs
set(knowhere_libs
knowhere
SPTAGLibStatic
arrow
jemalloc_pic
faiss
openblas
lapack
tbb
cudart
cublas
)
if (${BUILD_FAISS_WITH_MKL} STREQUAL "ON")
set(db_libs ${db_libs} ${MKL_LIBS} ${MKL_LIBS})
else ()
set(db_libs ${db_libs}
lapack
openblas)
endif ()
target_link_libraries(scheduler_test ${scheduler_libs} ${knowhere_libs} ${unittest_libs})
......
......@@ -34,9 +34,6 @@ cuda_add_executable(server_test
set(require_libs
knowhere
faiss
openblas
lapack
stdc++
cudart
cublas
......@@ -51,14 +48,6 @@ set(require_libs
pthread
)
if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON")
set(require_libs ${require_libs} ${MKL_LIBS} ${MKL_LIBS})
else()
set(require_libs ${require_libs}
lapack
openblas)
endif()
target_link_libraries(server_test
${require_libs}
${cuda_library}
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册