Skip to content

milvus
milvus

BaiXuePrincess / milvus
与 Fork 源项目一致

Fork自 milvus / milvus

通知 7
Star 4
Fork 0
milvus
milvus

前往新版Gitcode，体验更适合开发者的 AI 搜索 >>

提交 488c728f 编写于 作者: Y Yu Kun
浏览文件
操作

Merge remote-tracking branch 'upstream/branch-0.4.0' into branch-0.4.0 


Former-commit-id: e46da5c74207942b8fe5d8e92c1e00b140a41ad0
上级 c9347eb1 eaff4e5f
展开全部 隐藏空白更改
内联 并排
Showing with 245 addition and 539 deletion
cpp/CHANGELOG.md
浏览文件 @ 488c728f
......@@ -26,6 +26,9 @@ Please mark all change in change log and use the ticket from JIRA.
- MS-467 - mysql db test failed
- MS-470 - Drop index success, which table not created
- MS-471 - code coverage run failed
- MS-492 - Drop index failed if index have been created with index_type: FLAT
- MS-493 - Knowhere unittest crash
- MS-453 - GPU search error when nprobe set more than 1024
## Improvement
- MS-327 - Clean code for milvus
......@@ -84,9 +87,11 @@ Please mark all change in change log and use the ticket from JIRA.
- MS-455 - Distribute tasks by minimal cost in scheduler
- MS-460 - Put transport speed as weight when choosing neighbour to execute task
- MS-459 - Add cache for pick function in tasktable
- MS-476 - Improve search performance
- MS-482 - Change search stream transport to unary in grpc
- MS-487 - Define metric type in CreateTable
- MS-488 - Improve code format in scheduler
- MS-495 - cmake: integrated knowhere
## New Feature
- MS-343 - Implement ResourceMgr
......
cpp/CMakeLists.txt
浏览文件 @ 488c728f
......@@ -31,6 +31,7 @@ if(CMAKE_BUILD_TYPE STREQUAL "Release")
set(BUILD_TYPE "release")
else()
set(BUILD_TYPE "debug")
SET(CMAKE_VERBOSE_MAKEFILE on)
endif()
message(STATUS "Build type = ${BUILD_TYPE}")
......@@ -70,21 +71,21 @@ set(MILVUS_BINARY_DIR ${PROJECT_BINARY_DIR})
find_package(CUDA)
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES -arch sm_60 --expt-extended-lambda")
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g")
message("CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR}")
message("CUDA_NVCC_FLAGS=${CUDA_NVCC_FLAGS}")
message(STATUS "CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR}")
message(STATUS "CUDA_NVCC_FLAGS=${CUDA_NVCC_FLAGS}")
set(CMAKE_CXX_STANDARD 14)
set(CMAKE_CXX_STANDARD_REQUIRED on)
if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)")
message("building milvus_engine on x86 architecture")
message(STATUS "building milvus_engine on x86 architecture")
set(MILVUS_BUILD_ARCH x86_64)
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "(ppc)")
message("building milvus_engine on ppc architecture")
message(STATUS "building milvus_engine on ppc architecture")
set(MILVUS_BUILD_ARCH ppc64le)
else()
message("unknown processor type")
message("CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}")
message(WARNING "unknown processor type")
message(WARNING "CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}")
set(MILVUS_BUILD_ARCH unknown)
endif()
......@@ -94,6 +95,8 @@ else()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -DELPP_THREAD_SAFE -fopenmp")
endif()
set(ALLOW_DUPLICATE_CUSTOM_TARGETS TRUE)
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
include(ExternalProject)
......@@ -104,14 +107,6 @@ include(ThirdPartyPackages)
include_directories(${MILVUS_SOURCE_DIR})
link_directories(${MILVUS_BINARY_DIR})
if (NOT DEFINED KNOWHERE_BUILD_DIR)
message(FATAL_ERROR "You must set environment variable KNOWHERE_BUILD_DIR")
endif()
message(STATUS "Build with ${KNOWHERE_BUILD_DIR}")
include_directories(${KNOWHERE_BUILD_DIR}/include)
include_directories(${KNOWHERE_BUILD_DIR}/include/SPTAG/AnnService)
link_directories(${KNOWHERE_BUILD_DIR}/lib)
## Following should be check
set(MILVUS_ENGINE_INCLUDE ${PROJECT_SOURCE_DIR}/include)
......@@ -119,7 +114,7 @@ set(MILVUS_ENGINE_SRC ${PROJECT_SOURCE_DIR}/src)
add_compile_definitions(PROFILER=${PROFILER})
message("MILVUS_ENABLE_PROFILING = ${MILVUS_ENABLE_PROFILING}")
message(STATUS "MILVUS_ENABLE_PROFILING = ${MILVUS_ENABLE_PROFILING}")
if (MILVUS_ENABLE_PROFILING STREQUAL "ON")
ADD_DEFINITIONS(-DMILVUS_ENABLE_PROFILING)
endif()
......
cpp/build.sh
浏览文件 @ 488c728f
......@@ -83,10 +83,6 @@ if [[ ! -d cmake_build ]]; then
MAKE_CLEAN="ON"
fi
pushd `pwd`/src/core
./build.sh ${KNOWHERE_OPTIONS}
popd
cd cmake_build
CUDA_COMPILER=/usr/local/cuda/bin/nvcc
......
cpp/src/CMakeLists.txt
浏览文件 @ 488c728f
......@@ -4,6 +4,13 @@
# Proprietary and confidential.
#-------------------------------------------------------------------------------
add_subdirectory(core)
set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE)
foreach(dir ${CORE_INCLUDE_DIRS})
include_directories(${dir})
endforeach()
aux_source_directory(cache cache_files)
aux_source_directory(config config_files)
aux_source_directory(server server_files)
......@@ -87,12 +94,6 @@ set(client_grpc_lib
set(third_party_libs
knowhere
SPTAGLibStatic
arrow
jemalloc_pic
faiss
openblas
lapack
easyloggingpp
sqlite
${client_grpc_lib}
......@@ -115,15 +116,6 @@ set(third_party_libs
cudart
)
if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON")
set(third_party_libs ${third_party_libs}
${MKL_LIBS})
else()
set(third_party_libs ${third_party_libs}
lapack
openblas)
endif()
if (MILVUS_ENABLE_PROFILING STREQUAL "ON")
set(third_party_libs ${third_party_libs}
gperftools
......@@ -171,13 +163,6 @@ set(server_libs
set(knowhere_libs
knowhere
SPTAGLibStatic
arrow
jemalloc_pic
faiss
openblas
lapack
tbb
)
add_executable(milvus_server
......@@ -195,8 +180,8 @@ target_link_libraries(milvus_server ${server_libs} ${knowhere_libs} ${third_part
install(TARGETS milvus_server DESTINATION bin)
install(FILES
${KNOWHERE_BUILD_DIR}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}tbb${CMAKE_SHARED_LIBRARY_SUFFIX}
${KNOWHERE_BUILD_DIR}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}tbb${CMAKE_SHARED_LIBRARY_SUFFIX}.2
${CMAKE_SOURCE_DIR}/src/core/thirdparty/tbb/${CMAKE_SHARED_LIBRARY_PREFIX}tbb${CMAKE_SHARED_LIBRARY_SUFFIX}
${CMAKE_SOURCE_DIR}/src/core/thirdparty/tbb/${CMAKE_SHARED_LIBRARY_PREFIX}tbb${CMAKE_SHARED_LIBRARY_SUFFIX}.2
${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}
${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3
${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3.2.4
......
cpp/src/core/CMakeLists.txt
浏览文件 @ 488c728f
......@@ -5,6 +5,7 @@
#-------------------------------------------------------------------------------
cmake_minimum_required(VERSION 3.14)
message(STATUS "---------------core--------------")
message(STATUS "Building using CMake version: ${CMAKE_VERSION}")
set(KNOWHERE_VERSION "0.1.0")
......@@ -38,18 +39,16 @@ endif()
MESSAGE(STATUS "CMAKE_CXX_FLAGS" ${CMAKE_CXX_FLAGS})
find_package(CUDA)
#set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES --expt-extended-lambda")
#set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g")
if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)")
message("building milvus_engine on x86 architecture")
message(STATUS "building milvus_engine on x86 architecture")
set(KNOWHERE_BUILD_ARCH x86_64)
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "(ppc)")
message("building milvus_engine on ppc architecture")
message(STATUS "building milvus_engine on ppc architecture")
set(KNOWHERE_BUILD_ARCH ppc64le)
else()
message("unknown processor type")
message("CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}")
message(WARNING "unknown processor type")
message(WARNING "CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}")
set(KNOWHERE_BUILD_ARCH unknown)
endif()
......@@ -60,14 +59,21 @@ else()
endif()
message(STATUS "Build type = ${BUILD_TYPE}")
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
set(CORE_SOURCE_DIR ${PROJECT_SOURCE_DIR})
set(CORE_BINARY_DIR ${PROJECT_BINARY_DIR})
message(STATUS "Core source dir: ${PROJECT_SOURCE_DIR}")
message(STATUS "Core binary dir: ${PROJECT_BINARY_DIR}")
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CORE_SOURCE_DIR}/cmake")
include(ExternalProject)
include(DefineOptions)
include(BuildUtils)
include(ThirdPartyPackages)
include(DefineOptionsCore)
include(BuildUtilsCore)
include(ThirdPartyPackagesCore)
add_subdirectory(src)
set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE)
if(BUILD_UNIT_TEST STREQUAL "ON")
add_subdirectory(test)
endif()
......
cpp/src/core/cmake/BuildUtils.cmake → cpp/src/core/cmake/BuildUtilsCore.cmake
浏览文件 @ 488c728f
......@@ -75,7 +75,7 @@ function(ExternalProject_Use_Cache project_name package_file install_path)
"Extracting ${package_file} to ${install_path}"
COMMAND
${CMAKE_COMMAND} -E tar xzvf ${package_file} ${install_path}
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
WORKING_DIRECTORY ${CORE_BINARY_DIR}
)
ExternalProject_Add_StepTargets(${project_name} extract)
......
cpp/src/core/cmake/DefineOptions.cmake → cpp/src/core/cmake/DefineOptionsCore.cmake
浏览文件 @ 488c728f
......@@ -104,7 +104,7 @@ macro(config_summary)
message(STATUS " Source directory: ${CMAKE_CURRENT_SOURCE_DIR}")
if(${CMAKE_EXPORT_COMPILE_COMMANDS})
message(
STATUS " Compile commands: ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json")
STATUS " Compile commands: ${CORE_BINARY_DIR}/compile_commands.json")
endif()
foreach(category ${KNOWHERE_OPTION_CATEGORIES})
......
cpp/src/core/include/knowhere/index/vector_index/gpu_ivf.h
浏览文件 @ 488c728f
......@@ -9,7 +9,7 @@ namespace zilliz {
namespace knowhere {
struct Resource {
Resource(std::shared_ptr<faiss::gpu::StandardGpuResources> &r): faiss_res(r) {
explicit Resource(std::shared_ptr<faiss::gpu::StandardGpuResources> &r): faiss_res(r) {
static int64_t global_id = 0;
id = global_id++;
}
......@@ -32,6 +32,11 @@ class FaissGpuResourceMgr {
static FaissGpuResourceMgr &
GetInstance();
// Free gpu resource, avoid cudaGetDevice error when deallocate.
// this func should be invoke before main return
void
Free();
void
AllocateTempMem(ResPtr &resource, const int64_t& device_id, const int64_t& size);
......
cpp/src/core/src/CMakeLists.txt
浏览文件 @ 488c728f
set(TBB_DIR ${CMAKE_SOURCE_DIR}/thirdparty/tbb)
set(TBB_DIR ${CORE_SOURCE_DIR}/thirdparty/tbb)
set(TBB_LIBRARIES ${TBB_DIR}/libtbb.so)
include_directories(${TBB_DIR}/include)
include_directories(${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
link_directories(${CUDA_TOOLKIT_ROOT_DIR}/lib64)
include_directories(${CMAKE_SOURCE_DIR}/include)
include_directories(${CMAKE_SOURCE_DIR}/thirdparty)
include_directories(${CMAKE_SOURCE_DIR}/thirdparty/SPTAG/AnnService)
include_directories(${CMAKE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include)
include_directories(${CORE_SOURCE_DIR}/include)
include_directories(${CORE_SOURCE_DIR}/thirdparty)
include_directories(${CORE_SOURCE_DIR}/thirdparty/SPTAG/AnnService)
include_directories(${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include)
set(SPTAG_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../thirdparty/SPTAG)
set(SPTAG_SOURCE_DIR ${CORE_SOURCE_DIR}/thirdparty/SPTAG)
file(GLOB HDR_FILES
${SPTAG_SOURCE_DIR}/AnnService/inc/Core/*.h
${SPTAG_SOURCE_DIR}/AnnService/inc/Core/Common/*.h
......@@ -24,9 +24,9 @@ file(GLOB SRC_FILES
${SPTAG_SOURCE_DIR}/AnnService/src/Core/KDT/*.cpp
${SPTAG_SOURCE_DIR}/AnnService/src/Helper/*.cpp)
#add_library(SPTAGLib SHARED ${SRC_FILES} ${HDR_FILES})
#target_link_libraries(SPTAGLib ${TBB_LIBRARIES})
add_library(SPTAGLibStatic STATIC ${SRC_FILES} ${HDR_FILES})
if(NOT TARGET SPTAGLibStatic)
add_library(SPTAGLibStatic STATIC ${SRC_FILES} ${HDR_FILES})
endif()
set(external_srcs
knowhere/adapter/sptag.cpp
......@@ -51,14 +51,13 @@ set(index_srcs
)
set(depend_libs
# libtcmalloc.a
SPTAGLibStatic
${TBB_LIBRARIES}
faiss
openblas
lapack
arrow
jemalloc_pic
${ARROW_PREFIX}/lib/libjemalloc_pic.a
cudart
cublas
gomp
......@@ -66,30 +65,19 @@ set(depend_libs
pthread
)
add_library(
knowhere STATIC
${external_srcs}
${index_srcs}
)
#target_compile_options(knowhere PUBLIC "-fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free")
if(NOT TARGET knowhere)
add_library(
knowhere STATIC
${external_srcs}
${index_srcs}
)
endif()
target_link_libraries(
knowhere
${depend_libs}
)
#add_library(
# knowhereS SHARED
# ${external_srcs}
# ${index_srcs}
#)
#
#target_link_libraries(
# knowhereS
## ${TBB_LIBRARIES}
# ${depend_libs}
#)
INSTALL(TARGETS
knowhere
SPTAGLibStatic
......@@ -98,7 +86,6 @@ INSTALL(TARGETS
INSTALL(FILES
${ARROW_STATIC_LIB}
# ${PARQUET_STATIC_LIB}
${ARROW_PREFIX}/lib/libjemalloc_pic.a
${FAISS_STATIC_LIB}
${LAPACK_STATIC_LIB}
......@@ -112,24 +99,35 @@ INSTALL(FILES ${OPENBLAS_REAL_STATIC_LIB}
DESTINATION lib
)
INSTALL(FILES ${CMAKE_SOURCE_DIR}/thirdparty/tbb/libtbb.so.2
# RENAME "libtbb.so.2"
INSTALL(FILES ${CORE_SOURCE_DIR}/thirdparty/tbb/libtbb.so.2
DESTINATION lib
)
INSTALL(FILES ${CMAKE_SOURCE_DIR}/thirdparty/tbb/libtbb.so
# RENAME "libtbb.so"
INSTALL(FILES ${CORE_SOURCE_DIR}/thirdparty/tbb/libtbb.so
DESTINATION lib
)
set(CORE_INCLUDE_DIRS
${CORE_SOURCE_DIR}/include
${CORE_SOURCE_DIR}/thirdparty
${CORE_SOURCE_DIR}/thirdparty/SPTAG/AnnService
${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include
${ARROW_INCLUDE_DIR}
${FAISS_INCLUDE_DIR}
${OPENBLAS_INCLUDE_DIR}
${LAPACK_INCLUDE_DIR}
${CORE_SOURCE_DIR}/thirdparty/tbb/include
)
set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE)
INSTALL(DIRECTORY
${CMAKE_SOURCE_DIR}/include/knowhere
${CMAKE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons
${CMAKE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons_ext
${CORE_SOURCE_DIR}/include/knowhere
${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons
${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons_ext
${ARROW_INCLUDE_DIR}/arrow
# ${ARROW_INCLUDE_DIR}/parquet
${FAISS_PREFIX}/include/faiss
${OPENBLAS_INCLUDE_DIR}/
${CMAKE_SOURCE_DIR}/thirdparty/tbb/include/tbb
${CORE_SOURCE_DIR}/thirdparty/tbb/include/tbb
DESTINATION
include)
......
cpp/src/core/src/knowhere/index/vector_index/gpu_ivf.cpp
浏览文件 @ 488c728f
......@@ -282,7 +282,7 @@ void FaissGpuResourceMgr::InitResource() {
for(auto& device : devices_params_) {
auto& resource_vec = idle_[device.first];
for (int i = 0; i < device.second.resource_num; ++i) {
for (int64_t i = 0; i < device.second.resource_num; ++i) {
auto res = std::make_shared<faiss::gpu::StandardGpuResources>();
// TODO(linxj): enable set pinned memory
......@@ -351,6 +351,18 @@ void FaissGpuResourceMgr::MoveToIdle(const int64_t &device_id, const ResPtr &res
idle_[device_id].insert(it, res);
}
void FaissGpuResourceMgr::Free() {
for (auto &item : in_use_) {
auto& res_vec = item.second;
res_vec.clear();
}
for (auto &item : idle_) {
auto& res_vec = item.second;
res_vec.clear();
}
is_init = false;
}
void GPUIndex::SetGpuDevice(const int &gpu_id) {
gpu_id_ = gpu_id;
}
......
cpp/src/core/test/CMakeLists.txt
浏览文件 @ 488c728f
include_directories(${CMAKE_SOURCE_DIR}/thirdparty)
include_directories(${CMAKE_SOURCE_DIR}/thirdparty/SPTAG/AnnService)
include_directories(${CMAKE_SOURCE_DIR}/include)
include_directories(${CMAKE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include)
include_directories(${CORE_SOURCE_DIR}/thirdparty)
include_directories(${CORE_SOURCE_DIR}/thirdparty/SPTAG/AnnService)
include_directories(${CORE_SOURCE_DIR}/include)
include_directories(${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include)
include_directories(/usr/local/cuda/include)
link_directories(/usr/local/cuda/lib64)
link_directories(${CMAKE_SOURCE_DIR}/thirdparty/tbb)
link_directories(${CORE_SOURCE_DIR}/thirdparty/tbb)
set(unittest_libs
gtest gmock gtest_main gmock_main)
message(STATUS "arrow prefix: ${ARROW_PREFIX}")
message(STATUS "libjemalloc_pic path: ${ARROW_PREFIX}/lib/libjemalloc_pic.a")
set(depend_libs
faiss openblas lapack
arrow jemalloc_pic
arrow "${ARROW_PREFIX}/lib/libjemalloc_pic.a"
tbb
)
......@@ -22,57 +24,55 @@ set(basic_libs
#<IVF-TEST>
#<<<<<<<<<<
set(ivf_srcs
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/common/exception.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/common/timer.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp
${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp
${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp
${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp
utils.cpp
)
add_executable(test_ivf test_ivf.cpp ${ivf_srcs})
if(NOT TARGET test_ivf)
add_executable(test_ivf test_ivf.cpp ${ivf_srcs})
endif()
target_link_libraries(test_ivf ${depend_libs} ${unittest_libs} ${basic_libs})
#<<<<<<<<<<
#<IDMAP-TEST>
#<<<<<<<<<<
set(idmap_srcs
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/common/exception.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/common/timer.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp
${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp
${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp
${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp
utils.cpp
)
add_executable(test_idmap test_idmap.cpp ${idmap_srcs})
if(NOT TARGET test_idmap)
add_executable(test_idmap test_idmap.cpp ${idmap_srcs})
endif()
target_link_libraries(test_idmap ${depend_libs} ${unittest_libs} ${basic_libs})
#<<<<<<<<<<
#<KDT-TEST>
#<<<<<<<<<<
set(kdt_srcs
${CMAKE_SOURCE_DIR}/src/knowhere/index/preprocessor/normalize.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/kdt_parameters.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/cpu_kdt_rng.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/adapter/sptag.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/common/exception.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/adapter/arrow.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/common/timer.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/preprocessor/normalize.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/kdt_parameters.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cpu_kdt_rng.cpp
${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp
${CORE_SOURCE_DIR}/src/knowhere/adapter/sptag.cpp
${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp
${CORE_SOURCE_DIR}/src/knowhere/adapter/arrow.cpp
${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp
utils.cpp
)
add_executable(test_kdt test_kdt.cpp ${kdt_srcs})
if(NOT TARGET test_kdt)
add_executable(test_kdt test_kdt.cpp ${kdt_srcs})
endif()
target_link_libraries(test_kdt
SPTAGLibStatic
${depend_libs} ${unittest_libs} ${basic_libs})
#<<<<<<<<<<
add_subdirectory(faiss_ori)
add_subdirectory(test_nsg)
......
cpp/src/core/test/faiss_ori/CMakeLists.txt
浏览文件 @ 488c728f
include_directories(${CMAKE_SOURCE_DIR}/thirdparty)
include_directories(${CMAKE_SOURCE_DIR}/include)
include_directories(${CORE_SOURCE_DIR}/thirdparty)
include_directories(${CORE_SOURCE_DIR}/include)
include_directories(/usr/local/cuda/include)
link_directories(/usr/local/cuda/lib64)
link_directories(${CMAKE_SOURCE_DIR}/thirdparty/tbb)
link_directories(${CORE_SOURCE_DIR}/thirdparty/tbb)
set(unittest_libs
gtest gmock gtest_main gmock_main)
set(depend_libs
faiss openblas lapack
arrow jemalloc_pic
arrow ${ARROW_PREFIX}/lib/libjemalloc_pic.a
tbb
)
......@@ -20,7 +20,7 @@ set(basic_libs
#<GPU-TEST>
#<<<<<<<<<<
add_executable(test_gpu gpuresource_test.cpp)
if(NOT TARGET test_gpu)
add_executable(test_gpu gpuresource_test.cpp)
endif()
target_link_libraries(test_gpu ${depend_libs} ${unittest_libs} ${basic_libs})
#<<<<<<<<<<
cpp/src/core/test/test_idmap.cpp
浏览文件 @ 488c728f
......@@ -26,6 +26,11 @@ class IDMAPTest : public DataGen, public ::testing::Test {
Init_with_default();
index_ = std::make_shared<IDMAP>();
}
void TearDown() override {
FaissGpuResourceMgr::GetInstance().Free();
}
protected:
IDMAPPtr index_ = nullptr;
};
......
cpp/src/core/test/test_ivf.cpp
浏览文件 @ 488c728f
......@@ -7,13 +7,11 @@
#include <gtest/gtest.h>
#include <iostream>
#include <sstream>
#include <thread>
#include <faiss/AutoTune.h>
#include <faiss/gpu/GpuAutoTune.h>
#include <faiss/gpu/GpuIndexIVFFlat.h>
#include <faiss/gpu/GpuClonerOptions.h>
#include "knowhere/index/vector_index/gpu_ivf.h"
#include "knowhere/index/vector_index/ivf.h"
......@@ -58,6 +56,9 @@ class IVFTest
index_ = IndexFactory(index_type);
FaissGpuResourceMgr::GetInstance().InitDevice(device_id, 1024*1024*200, 1024*1024*300, 2);
}
void TearDown() override {
FaissGpuResourceMgr::GetInstance().Free();
}
protected:
std::string index_type;
......@@ -369,6 +370,7 @@ class GPURESTEST
void TearDown() override {
delete ids;
delete dis;
FaissGpuResourceMgr::GetInstance().Free();
}
protected:
......
cpp/src/core/test/test_nsg/CMakeLists.txt
浏览文件 @ 488c728f
......@@ -2,7 +2,6 @@
include_directories(/usr/local/include/gperftools)
link_directories(/usr/local/lib)
#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free")
add_definitions(-std=c++11 -O3 -lboost -march=native -Wall -DINFO)
find_package(OpenMP)
......@@ -14,37 +13,29 @@ else ()
endif ()
message(${OpenMP_CXX_FLAGS})
include_directories(${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg)
#include_directories(/home/zilliz/opt/app/pyenv/versions/3.6.8/include/python3.6m)
#include_directories(/home/zilliz/opt/app/pyenv/versions/3.6.8/envs/megasearch_testframework_dev/lib/python3.6/site-packages/numpy/core/include)
include_directories(${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg)
aux_source_directory(${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg nsg_src)
#add_library(nsg_raw SHARED ${nsg_src})
#target_link_libraries(nsg_raw
# gomp)
aux_source_directory(${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg nsg_src)
set(interface_src
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg_index.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/common/exception.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/common/timer.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg_index.cpp
${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp
${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp
${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp
../utils.cpp
)
add_executable(test_nsg
test_nsg.cpp
${interface_src}
${nsg_src}
)
#target_link_libraries(test_nsg
# # libprofiler.so
## -ltcmalloc
## gomp
# )
if(NOT TARGET test_nsg)
add_executable(test_nsg
test_nsg.cpp
${interface_src}
${nsg_src}
)
endif()
target_link_libraries(test_nsg ${depend_libs} ${unittest_libs} ${basic_libs})
##############################
\ No newline at end of file
cpp/src/server/Server.cpp
浏览文件 @ 488c728f
......@@ -19,6 +19,7 @@
#include <unistd.h>
#include <string.h>
#include <src/scheduler/SchedInst.h>
#include "knowhere/index/vector_index/gpu_ivf.h"
#include "metrics/Metrics.h"
#include "DBWrapper.h"
......@@ -232,6 +233,7 @@ Server::StopService() {
grpc::GrpcMilvusServer::StopService();
DBWrapper::GetInstance().StopService();
engine::StopSchedulerService();
knowhere::FaissGpuResourceMgr::GetInstance().Free(); // free gpu resource.
}
}
......
cpp/src/server/grpc_impl/GrpcRequestTask.cpp
浏览文件 @ 488c728f
......@@ -952,15 +952,8 @@ DropIndexTask::OnExecute() {
return SetError(res, "Invalid table name: " + table_name_);
}
//step 2:check index existence
engine::TableIndex index;
engine::Status stat = DBWrapper::DB()->DescribeIndex(table_name_, index);
if (index.engine_type_ == 1) {
return SetError(SERVER_UNEXPECTED_ERROR, "index not existed");
}
//step 3: check table existence
stat = DBWrapper::DB()->DropIndex(table_name_);
//step 2: check table existence
auto stat = DBWrapper::DB()->DropIndex(table_name_);
if (!stat.ok()) {
return SetError(DB_META_TRANSACTION_FAILED, stat.ToString());
}
......
cpp/src/wrapper/knowhere/vec_impl.cpp
浏览文件 @ 488c728f
......@@ -72,8 +72,11 @@ server::KnowhereError VecIndexImpl::Search(const long &nq, const float *xq, floa
auto k = cfg["k"].as<int>();
auto dataset = GenDataset(nq, dim, xq);
Config search_cfg;
auto res = index_->Search(dataset, cfg);
Config search_cfg = cfg;
ParameterValidation(type, search_cfg);
auto res = index_->Search(dataset, search_cfg);
auto ids_array = res->array()[0];
auto dis_array = res->array()[1];
......
cpp/src/wrapper/knowhere/vec_index.cpp
浏览文件 @ 488c728f
......@@ -71,7 +71,7 @@ size_t FileIOWriter::operator()(void *ptr, size_t size) {
}
VecIndexPtr GetVecIndexFactory(const IndexType &type, const Config& cfg) {
VecIndexPtr GetVecIndexFactory(const IndexType &type, const Config &cfg) {
std::shared_ptr<zilliz::knowhere::VectorIndex> index;
auto gpu_device = cfg.get_with_default("gpu_id", 0);
switch (type) {
......@@ -235,6 +235,31 @@ void AutoGenParams(const IndexType &type, const long &size, zilliz::knowhere::Co
}
}
#if CUDA_VERSION > 9000
#define GPU_MAX_NRPOBE 2048
#else
#define GPU_MAX_NRPOBE 1024
#endif
void ParameterValidation(const IndexType &type, Config &cfg) {
switch (type) {
case IndexType::FAISS_IVFSQ8_GPU:
case IndexType::FAISS_IVFFLAT_GPU:
case IndexType::FAISS_IVFPQ_GPU: {
if (cfg.get_with_default("nprobe", 0) != 0) {
auto nprobe = cfg["nprobe"].as<int>();
if (nprobe > GPU_MAX_NRPOBE) {
WRAPPER_LOG_WARNING << "When search with GPU, nprobe shoud be no more than " << GPU_MAX_NRPOBE << ", but you passed " << nprobe
<< ". Search with " << GPU_MAX_NRPOBE << " instead";
cfg.insert_or_assign("nprobe", GPU_MAX_NRPOBE);
}
}
break;
}
default:break;
}
}
IndexType ConvertToCpuIndexType(const IndexType &type) {
// TODO(linxj): add IDMAP
switch (type) {
......
cpp/src/wrapper/knowhere/vec_index.h
浏览文件 @ 488c728f
......@@ -14,6 +14,8 @@
#include "knowhere/common/config.h"
#include "knowhere/common/binary_set.h"
#include "cuda.h"
namespace zilliz {
namespace milvus {
......@@ -90,6 +92,8 @@ extern VecIndexPtr LoadVecIndex(const IndexType &index_type, const zilliz::knowh
extern void AutoGenParams(const IndexType& type, const long& size, Config& cfg);
extern void ParameterValidation(const IndexType& type, Config& cfg);
extern IndexType ConvertToCpuIndexType(const IndexType& type);
extern IndexType ConvertToGpuIndexType(const IndexType& type);
......
cpp/unittest/CMakeLists.txt
浏览文件 @ 488c728f
......@@ -38,9 +38,13 @@ set(unittest_libs
cudart
)
foreach(dir ${CORE_INCLUDE_DIRS})
include_directories(${dir})
endforeach()
add_subdirectory(server)
add_subdirectory(db)
#add_subdirectory(knowhere)
add_subdirectory(knowhere)
add_subdirectory(metrics)
#add_subdirectory(scheduler)
#add_subdirectory(storage)
\ No newline at end of file
cpp/unittest/db/CMakeLists.txt
浏览文件 @ 488c728f
......@@ -35,8 +35,6 @@ link_directories("/usr/local/cuda/lib64")
include_directories(/usr/include/mysql)
#add_definitions(-DBOOST_ERROR_CODE_HEADER_ONLY)
set(db_test_src
${config_files}
${cache_srcs}
......@@ -69,25 +67,10 @@ set(db_libs
set(knowhere_libs
knowhere
SPTAGLibStatic
arrow
jemalloc_pic
faiss
openblas
lapack
tbb
cudart
cublas
)
if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON")
set(db_libs ${db_libs} ${MKL_LIBS} ${MKL_LIBS})
else()
set(db_libs ${db_libs}
lapack
openblas)
endif()
target_link_libraries(db_test ${db_libs} ${knowhere_libs} ${unittest_libs})
install(TARGETS db_test DESTINATION unittest)
......
cpp/unittest/knowhere/CMakeLists.txt
浏览文件 @ 488c728f
......@@ -8,13 +8,6 @@ set(helper
set(knowhere_libs
knowhere
SPTAGLibStatic
arrow
jemalloc_pic
faiss
openblas
lapack
tbb
cudart
cublas
)
......
cpp/unittest/knowhere/knowhere_test.cpp
浏览文件 @ 488c728f
......@@ -40,6 +40,9 @@ class KnowhereWrapperTest
index_ = GetVecIndexFactory(index_type);
}
void TearDown() override {
zilliz::knowhere::FaissGpuResourceMgr::GetInstance().Free();
}
void AssertResult(const std::vector<long> &ids, const std::vector<float> &dis) {
EXPECT_EQ(ids.size(), nq * k);
......
cpp/unittest/metrics/CMakeLists.txt
浏览文件 @ 488c728f
#set(CURL_LIBRARY "-lcurl")
#find_package(CURL REQUIRED)
#INCLUDE_DIRECTORIES(${CURL_INCLUDE_DIR})
#
#INCLUDE_DIRECTORIES(~/development/lib/usr/local/include)
#LINK_DIRECTORIES(~/development/lib/usr/local/lib)
include_directories(../../src)
......@@ -72,13 +62,6 @@ add_executable(metrics_test ${count_test_src} ${require_files} )
set(knowhere_libs
knowhere
SPTAGLibStatic
arrow
jemalloc_pic
faiss
openblas
lapack
tbb
)
target_link_libraries(metrics_test
......@@ -96,12 +79,5 @@ target_link_libraries(metrics_test
mysqlpp
${unittest_libs}
)
if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON")
target_link_libraries(metrics_test ${MKL_LIBS} ${MKL_LIBS})
else()
target_link_libraries(metrics_test
lapack
openblas)
endif()
install(TARGETS metrics_test DESTINATION unittest)
\ No newline at end of file
cpp/unittest/scheduler/CMakeLists.txt
浏览文件 @ 488c728f
......@@ -33,8 +33,6 @@ link_directories("/usr/local/cuda/lib64")
include_directories(/usr/include/mysql)
#add_definitions(-DBOOST_ERROR_CODE_HEADER_ONLY)
set(scheduler_test_src
${unittest_srcs}
${test_srcs}
......@@ -69,25 +67,10 @@ set(scheduler_libs
set(knowhere_libs
knowhere
SPTAGLibStatic
arrow
jemalloc_pic
faiss
openblas
lapack
tbb
cudart
cublas
)
if (${BUILD_FAISS_WITH_MKL} STREQUAL "ON")
set(db_libs ${db_libs} ${MKL_LIBS} ${MKL_LIBS})
else ()
set(db_libs ${db_libs}
lapack
openblas)
endif ()
target_link_libraries(scheduler_test ${scheduler_libs} ${knowhere_libs} ${unittest_libs})
......
cpp/unittest/server/CMakeLists.txt
浏览文件 @ 488c728f
......@@ -34,9 +34,6 @@ cuda_add_executable(server_test
set(require_libs
knowhere
faiss
openblas
lapack
stdc++
cudart
cublas
......@@ -51,14 +48,6 @@ set(require_libs
pthread
)
if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON")
set(require_libs ${require_libs} ${MKL_LIBS} ${MKL_LIBS})
else()
set(require_libs ${require_libs}
lapack
openblas)
endif()
target_link_libraries(server_test
${require_libs}
${cuda_library}
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或