未验证 提交 52936073 编写于 作者: C Cai Yudong 提交者: GitHub

Caiyd support disable sptag (#2308)

* remove KNOWHERE_GPU_VERSION
Signed-off-by: Nyudong.cai <yudong.cai@zilliz.com>

* add MILVUS_SUPPORT_SPTAG
Signed-off-by: Nyudong.cai <yudong.cai@zilliz.com>

* fix SPTAG build issue
Signed-off-by: Nyudong.cai <yudong.cai@zilliz.com>

* update changelog
Signed-off-by: Nyudong.cai <yudong.cai@zilliz.com>

* update changelog
Signed-off-by: Nyudong.cai <yudong.cai@zilliz.com>

* fix unittest
Signed-off-by: Nyudong.cai <yudong.cai@zilliz.com>

* update changelog
Signed-off-by: Nyudong.cai <yudong.cai@zilliz.com>
上级 03c106cd
......@@ -9,6 +9,7 @@ Please mark all change in change log and use the issue from GitHub
## Feature
## Improvement
- \#2307 Disable index SPTAG by default
## Task
......@@ -38,7 +39,7 @@ Please mark all change in change log and use the issue from GitHub
- \#2231 Use server_config to define hard-delete delay time for segment files
- \#2261 Re-define result returned by has_collection if collection in delete state
- \#2266 Server hang when using multi-clients to query different collections
- \#2280 has_partition should return true for '_default'
- \#2280 has_partition should return true for `_default`
## Feature
- \#1751 Add api SearchByID
......@@ -76,7 +77,6 @@ Please mark all change in change log and use the issue from GitHub
## Task
# Milvus 0.8.0 (2020-04-15)
## Bug
......
......@@ -207,6 +207,7 @@ ExecutionEngineImpl::CreatetVecIndex(EngineType type) {
index = vec_index_factory.CreateVecIndex(knowhere::IndexEnum::INDEX_NSG, mode);
break;
}
#ifdef MILVUS_SUPPORT_SPTAG
case EngineType::SPTAG_KDT: {
index = vec_index_factory.CreateVecIndex(knowhere::IndexEnum::INDEX_SPTAG_KDT_RNT, mode);
break;
......@@ -215,6 +216,7 @@ ExecutionEngineImpl::CreatetVecIndex(EngineType type) {
index = vec_index_factory.CreateVecIndex(knowhere::IndexEnum::INDEX_SPTAG_BKT_RNT, mode);
break;
}
#endif
case EngineType::HNSW: {
index = vec_index_factory.CreateVecIndex(knowhere::IndexEnum::INDEX_HNSW, mode);
break;
......
......@@ -11,29 +11,13 @@
# or implied. See the License for the specific language governing permissions and limitations under the License.
#-------------------------------------------------------------------------------
cmake_minimum_required(VERSION 3.12)
message(STATUS "------------------------------KNOWHERE-----------------------------------")
message(STATUS "Building using CMake version: ${CMAKE_VERSION}")
set(KNOWHERE_VERSION "0.6.0")
string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" KNOWHERE_BASE_VERSION "${KNOWHERE_VERSION}")
project(knowhere VERSION "${KNOWHERE_BASE_VERSION}" LANGUAGES C CXX)
project(knowhere LANGUAGES C CXX)
set(CMAKE_CXX_STANDARD 17)
set(KNOWHERE_VERSION_MAJOR "${knowhere_VERSION_MAJOR}")
set(KNOWHERE_VERSION_MINOR "${knowhere_VERSION_MINOR}")
set(KNOWHERE_VERSION_PATCH "${knowhere_VERSION_PATCH}")
if (KNOWHERE_VERSION_MAJOR STREQUAL ""
OR KNOWHERE_VERSION_MINOR STREQUAL ""
OR KNOWHERE_VERSION_PATCH STREQUAL "")
message(FATAL_ERROR "Failed to determine Knowhere version from '${KNOWHERE_VERSION}'")
endif ()
message(STATUS "Knowhere version: "
"${KNOWHERE_VERSION_MAJOR}.${KNOWHERE_VERSION_MINOR}.${KNOWHERE_VERSION_PATCH} "
"(full: '${KNOWHERE_VERSION}')")
# if no build build type is specified, default to release builds
if (NOT CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE Release)
......@@ -66,7 +50,7 @@ include(ExternalProject)
include(DefineOptionsCore)
include(BuildUtilsCore)
if (MILVUS_GPU_VERSION OR KNOWHERE_GPU_VERSION)
if (MILVUS_GPU_VERSION)
message(STATUS "Building Knowhere GPU version")
add_compile_definitions("MILVUS_GPU_VERSION")
enable_language(CUDA)
......@@ -76,16 +60,21 @@ else ()
message(STATUS "Building Knowhere CPU version")
endif ()
if (MILVUS_SUPPORT_SPTAG)
message(STATUS "Building Knowhere with SPTAG supported")
add_compile_definitions("MILVUS_SUPPORT_SPTAG")
endif ()
include(ThirdPartyPackagesCore)
if (CMAKE_BUILD_TYPE STREQUAL "Release")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp")
if (KNOWHERE_GPU_VERSION)
if (MILVUS_GPU_VERSION)
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O3")
endif ()
else ()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -DELPP_THREAD_SAFE -fopenmp")
if (KNOWHERE_GPU_VERSION)
if (MILVUS_GPU_VERSION)
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g")
endif ()
endif ()
......
#-------------------------------------------------------------------------------
# Copyright (C) 2019-2020 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under the License.
#-------------------------------------------------------------------------------
macro(set_option_category name)
set(KNOWHERE_OPTION_CATEGORY ${name})
......@@ -40,15 +52,6 @@ macro(define_option_string name description default)
endif ()
endmacro()
#----------------------------------------------------------------------
set_option_category("GPU version")
if (MILVUS_GPU_VERSION)
define_option(KNOWHERE_GPU_VERSION "Build GPU version" ON)
else ()
define_option(KNOWHERE_GPU_VERSION "Build GPU version" OFF)
endif ()
#----------------------------------------------------------------------
set_option_category("Thirdparty")
......
#-------------------------------------------------------------------------------
# Copyright (C) 2019-2020 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
......@@ -8,6 +9,7 @@
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under the License.
#-------------------------------------------------------------------------------
set(KNOWHERE_THIRDPARTY_DEPENDENCIES
Arrow
......@@ -532,7 +534,7 @@ macro(build_faiss)
endif()
endif ()
if (KNOWHERE_GPU_VERSION)
if (MILVUS_GPU_VERSION)
set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
"--with-cuda=${CUDA_TOOLKIT_ROOT_DIR}"
"--with-cuda-arch=-gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_61,code=sm_61 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75"
......
#-------------------------------------------------------------------------------
# Copyright (C) 2019-2020 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under the License.
#-------------------------------------------------------------------------------
include_directories(${INDEX_SOURCE_DIR}/knowhere)
include_directories(${INDEX_SOURCE_DIR}/thirdparty)
include_directories(${INDEX_SOURCE_DIR}/thirdparty/SPTAG/AnnService)
set(SPTAG_SOURCE_DIR ${INDEX_SOURCE_DIR}/thirdparty/SPTAG)
file(GLOB HDR_FILES
if (MILVUS_SUPPORT_SPTAG)
include_directories(${INDEX_SOURCE_DIR}/thirdparty/SPTAG/AnnService)
set(SPTAG_SOURCE_DIR ${INDEX_SOURCE_DIR}/thirdparty/SPTAG)
file(GLOB HDR_FILES
${SPTAG_SOURCE_DIR}/AnnService/inc/Core/*.h
${SPTAG_SOURCE_DIR}/AnnService/inc/Core/Common/*.h
${SPTAG_SOURCE_DIR}/AnnService/inc/Core/BKT/*.h
${SPTAG_SOURCE_DIR}/AnnService/inc/Core/KDT/*.h
${SPTAG_SOURCE_DIR}/AnnService/inc/Helper/*.h)
file(GLOB SRC_FILES
file(GLOB SRC_FILES
${SPTAG_SOURCE_DIR}/AnnService/src/Core/*.cpp
${SPTAG_SOURCE_DIR}/AnnService/src/Core/Common/*.cpp
${SPTAG_SOURCE_DIR}/AnnService/src/Core/BKT/*.cpp
${SPTAG_SOURCE_DIR}/AnnService/src/Core/KDT/*.cpp
${SPTAG_SOURCE_DIR}/AnnService/src/Helper/*.cpp)
if (NOT TARGET SPTAGLibStatic)
if (NOT TARGET SPTAGLibStatic)
add_library(SPTAGLibStatic STATIC ${SRC_FILES} ${HDR_FILES})
endif ()
endif ()
set(external_srcs
......@@ -27,11 +43,9 @@ set(external_srcs
)
set(index_srcs
knowhere/index/vector_index/adapter/SptagAdapter.cpp
knowhere/index/vector_index/adapter/VectorAdapter.cpp
knowhere/index/vector_index/helpers/FaissIO.cpp
knowhere/index/vector_index/helpers/IndexParameter.cpp
knowhere/index/vector_index/helpers/SPTAGParameterMgr.cpp
knowhere/index/vector_index/impl/nsg/Distance.cpp
knowhere/index/vector_index/impl/nsg/NSG.cpp
knowhere/index/vector_index/impl/nsg/NSGHelper.cpp
......@@ -48,19 +62,35 @@ set(index_srcs
knowhere/index/vector_index/IndexIVFPQ.cpp
knowhere/index/vector_index/IndexIVFSQ.cpp
knowhere/index/vector_index/IndexNSG.cpp
knowhere/index/vector_index/IndexSPTAG.cpp
knowhere/index/vector_index/IndexType.cpp
knowhere/index/vector_index/VecIndexFactory.cpp
knowhere/index/vector_index/IndexAnnoy.cpp
)
if (MILVUS_SUPPORT_SPTAG)
set(index_srcs
knowhere/index/vector_index/adapter/SptagAdapter.cpp
knowhere/index/vector_index/helpers/SPTAGParameterMgr.cpp
knowhere/index/vector_index/IndexSPTAG.cpp
${index_srcs}
)
endif ()
set(depend_libs
SPTAGLibStatic
faiss
gomp
gfortran
pthread
)
if (MILVUS_SUPPORT_SPTAG)
set(depend_libs
SPTAGLibStatic
${depend_libs}
)
endif ()
if (FAISS_WITH_MKL)
set(depend_libs ${depend_libs}
"-Wl,--start-group \
......@@ -76,7 +106,7 @@ else ()
)
endif ()
if (KNOWHERE_GPU_VERSION)
if (MILVUS_GPU_VERSION)
include_directories(${CUDA_INCLUDE_DIRS})
link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64")
set(cuda_lib
......@@ -96,7 +126,6 @@ if (KNOWHERE_GPU_VERSION)
knowhere/index/vector_index/helpers/Cloner.cpp
knowhere/index/vector_index/helpers/FaissGpuResourceMgr.cpp
)
endif ()
if (NOT TARGET knowhere)
......@@ -115,11 +144,16 @@ target_link_libraries(
set(INDEX_INCLUDE_DIRS
${INDEX_SOURCE_DIR}/knowhere
${INDEX_SOURCE_DIR}/thirdparty
${INDEX_SOURCE_DIR}/thirdparty/SPTAG/AnnService
# ${ARROW_INCLUDE_DIR}
${FAISS_INCLUDE_DIR}
${OpenBLAS_INCLUDE_DIR}
${LAPACK_INCLUDE_DIR}
)
if (MILVUS_SUPPORT_SPTAG)
set(INDEX_INCLUDE_DIRS
${INDEX_SOURCE_DIR}/thirdparty/SPTAG/AnnService
${INDEX_INCLUDE_DIRS}
)
endif ()
set(INDEX_INCLUDE_DIRS ${INDEX_INCLUDE_DIRS} PARENT_SCOPE)
......@@ -43,8 +43,10 @@ AdapterMgr::RegisterAdapter() {
REGISTER_CONF_ADAPTER(BinIDMAPConfAdapter, IndexEnum::INDEX_FAISS_BIN_IDMAP, idmap_bin_adapter);
REGISTER_CONF_ADAPTER(BinIDMAPConfAdapter, IndexEnum::INDEX_FAISS_BIN_IVFFLAT, ivf_bin_adapter);
REGISTER_CONF_ADAPTER(NSGConfAdapter, IndexEnum::INDEX_NSG, nsg_adapter);
#ifdef MILVUS_SUPPORT_SPTAG
REGISTER_CONF_ADAPTER(ConfAdapter, IndexEnum::INDEX_SPTAG_KDT_RNT, sptag_kdt_adapter);
REGISTER_CONF_ADAPTER(ConfAdapter, IndexEnum::INDEX_SPTAG_BKT_RNT, sptag_bkt_adapter);
#endif
REGISTER_CONF_ADAPTER(HNSWConfAdapter, IndexEnum::INDEX_HNSW, hnsw_adapter);
REGISTER_CONF_ADAPTER(ANNOYConfAdapter, IndexEnum::INDEX_ANNOY, annoy_adapter);
}
......
......@@ -31,8 +31,10 @@ static std::unordered_map<int32_t, std::string> old_index_type_str_map = {
{(int32_t)OldIndexType::FAISS_IVFSQ8_GPU, IndexEnum::INDEX_FAISS_IVFSQ8},
{(int32_t)OldIndexType::FAISS_IVFSQ8_HYBRID, IndexEnum::INDEX_FAISS_IVFSQ8H},
{(int32_t)OldIndexType::NSG_MIX, IndexEnum::INDEX_NSG},
#ifdef MILVUS_SUPPORT_SPTAG
{(int32_t)OldIndexType::SPTAG_KDT_RNT_CPU, IndexEnum::INDEX_SPTAG_KDT_RNT},
{(int32_t)OldIndexType::SPTAG_BKT_RNT_CPU, IndexEnum::INDEX_SPTAG_BKT_RNT},
#endif
{(int32_t)OldIndexType::HNSW, IndexEnum::INDEX_HNSW},
{(int32_t)OldIndexType::ANNOY, IndexEnum::INDEX_ANNOY},
{(int32_t)OldIndexType::FAISS_BIN_IDMAP, IndexEnum::INDEX_FAISS_BIN_IDMAP},
......@@ -47,8 +49,10 @@ static std::unordered_map<std::string, int32_t> str_old_index_type_map = {
{IndexEnum::INDEX_FAISS_IVFSQ8, (int32_t)OldIndexType::FAISS_IVFSQ8_CPU},
{IndexEnum::INDEX_FAISS_IVFSQ8H, (int32_t)OldIndexType::FAISS_IVFSQ8_HYBRID},
{IndexEnum::INDEX_NSG, (int32_t)OldIndexType::NSG_MIX},
#ifdef MILVUS_SUPPORT_SPTAG
{IndexEnum::INDEX_SPTAG_KDT_RNT, (int32_t)OldIndexType::SPTAG_KDT_RNT_CPU},
{IndexEnum::INDEX_SPTAG_BKT_RNT, (int32_t)OldIndexType::SPTAG_BKT_RNT_CPU},
#endif
{IndexEnum::INDEX_HNSW, (int32_t)OldIndexType::HNSW},
{IndexEnum::INDEX_ANNOY, (int32_t)OldIndexType::ANNOY},
{IndexEnum::INDEX_FAISS_BIN_IDMAP, (int32_t)OldIndexType::FAISS_BIN_IDMAP},
......@@ -66,8 +70,10 @@ const char* INDEX_FAISS_IVFSQ8H = "IVF_SQ8_HYBRID";
const char* INDEX_FAISS_BIN_IDMAP = "BIN_IDMAP";
const char* INDEX_FAISS_BIN_IVFFLAT = "BIN_IVF_FLAT";
const char* INDEX_NSG = "NSG";
#ifdef MILVUS_SUPPORT_SPTAG
const char* INDEX_SPTAG_KDT_RNT = "SPTAG_KDT_RNT";
const char* INDEX_SPTAG_BKT_RNT = "SPTAG_BKT_RNT";
#endif
const char* INDEX_HNSW = "HNSW";
const char* INDEX_ANNOY = "ANNOY";
} // namespace IndexEnum
......
......@@ -52,8 +52,10 @@ extern const char* INDEX_FAISS_IVFSQ8H;
extern const char* INDEX_FAISS_BIN_IDMAP;
extern const char* INDEX_FAISS_BIN_IVFFLAT;
extern const char* INDEX_NSG;
#ifdef MILVUS_SUPPORT_SPTAG
extern const char* INDEX_SPTAG_KDT_RNT;
extern const char* INDEX_SPTAG_BKT_RNT;
#endif
extern const char* INDEX_HNSW;
extern const char* INDEX_ANNOY;
} // namespace IndexEnum
......
......@@ -22,7 +22,9 @@
#include "knowhere/index/vector_index/IndexIVFPQ.h"
#include "knowhere/index/vector_index/IndexIVFSQ.h"
#include "knowhere/index/vector_index/IndexNSG.h"
#ifdef MILVUS_SUPPORT_SPTAG
#include "knowhere/index/vector_index/IndexSPTAG.h"
#endif
#ifdef MILVUS_GPU_VERSION
#include <cuda.h>
......@@ -73,10 +75,12 @@ VecIndexFactory::CreateVecIndex(const IndexType& type, const IndexMode mode) {
return std::make_shared<knowhere::BinaryIVF>();
} else if (type == IndexEnum::INDEX_NSG) {
return std::make_shared<knowhere::NSG>(-1);
#ifdef MILVUS_SUPPORT_SPTAG
} else if (type == IndexEnum::INDEX_SPTAG_KDT_RNT) {
return std::make_shared<knowhere::CPUSPTAGRNG>("KDT");
} else if (type == IndexEnum::INDEX_SPTAG_BKT_RNT) {
return std::make_shared<knowhere::CPUSPTAGRNG>("BKT");
#endif
} else if (type == IndexEnum::INDEX_HNSW) {
return std::make_shared<knowhere::IndexHNSW>();
} else if (type == IndexEnum::INDEX_ANNOY) {
......
......@@ -38,7 +38,7 @@ set(util_srcs
${INDEX_SOURCE_DIR}/unittest/utils.cpp
)
if (KNOWHERE_GPU_VERSION)
if (MILVUS_GPU_VERSION)
include_directories(${CUDA_INCLUDE_DIRS})
link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64")
set(cuda_lib
......@@ -63,7 +63,7 @@ set(faiss_srcs
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp
)
if (KNOWHERE_GPU_VERSION)
if (MILVUS_GPU_VERSION)
set(faiss_srcs ${faiss_srcs}
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu/IndexGPUIDMAP.cpp
......@@ -89,7 +89,7 @@ endif ()
target_link_libraries(test_knowhere_common ${depend_libs} ${unittest_libs} ${basic_libs})
install(TARGETS test_knowhere_common DESTINATION unittest)
if (KNOWHERE_GPU_VERSION)
if (MILVUS_GPU_VERSION)
################################################################################
#<GPU-RESOURCE-TEST>
add_executable(test_gpuresource test_gpuresource.cpp ${util_srcs} ${faiss_srcs})
......@@ -172,18 +172,20 @@ install(TARGETS test_hnsw DESTINATION unittest)
################################################################################
#<SPTAG-TEST>
set(sptag_srcs
if (MILVUS_SUPPORT_SPTAG)
set(sptag_srcs
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/adapter/SptagAdapter.cpp
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.cpp
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexSPTAG.cpp
)
if (NOT TARGET test_sptag)
if (NOT TARGET test_sptag)
add_executable(test_sptag test_sptag.cpp ${sptag_srcs} ${util_srcs})
endif ()
target_link_libraries(test_sptag
endif ()
target_link_libraries(test_sptag
SPTAGLibStatic
${depend_libs} ${unittest_libs} ${basic_libs})
install(TARGETS test_sptag DESTINATION unittest)
install(TARGETS test_sptag DESTINATION unittest)
endif ()
################################################################################
#<ANNOY-TEST>
......
if (KNOWHERE_GPU_VERSION)
if (MILVUS_GPU_VERSION)
include_directories(${INDEX_SOURCE_DIR}/thirdparty)
include_directories(${INDEX_SOURCE_DIR}/include)
......
if (KNOWHERE_GPU_VERSION)
if (MILVUS_GPU_VERSION)
include_directories(${INDEX_SOURCE_DIR}/thirdparty)
include_directories(${INDEX_SOURCE_DIR}/include)
......
......@@ -180,6 +180,7 @@ TEST_F(EngineTest, ENGINE_IMPL_TEST) {
#endif
}
#ifdef MILVUS_SUPPORT_SPTAG
{
milvus::json index_params = {{"nlist", 10}};
auto engine_ptr = CreateExecEngine(index_params);
......@@ -191,6 +192,7 @@ TEST_F(EngineTest, ENGINE_IMPL_TEST) {
auto status = engine_ptr->CopyToCpu();
ASSERT_FALSE(status.ok());
}
#endif
#ifdef MILVUS_GPU_VERSION
{
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册