From 5bcb660110bf42daf01d3de5f9b58e8653366a17 Mon Sep 17 00:00:00 2001 From: "xiaojun.lin" Date: Fri, 11 Oct 2019 19:41:43 +0800 Subject: [PATCH] MS-635 add compile option Former-commit-id: 773d61d34bdb131ae3219ddfbe9e4eb656292230 --- cpp/CHANGELOG.md | 1 + cpp/CMakeLists.txt | 4 ++ cpp/build.sh | 9 +++- .../core/cmake/ThirdPartyPackagesCore.cmake | 25 +++++---- .../index/vector_index/FaissBaseIndex.cpp | 8 +-- .../index/vector_index/IndexIVFSQ.cpp | 5 ++ .../index/vector_index/IndexIVFSQHybrid.cpp | 54 +++++++++++++++++++ .../index/vector_index/IndexIVFSQHybrid.h | 2 + cpp/src/core/unittest/test_ivf.cpp | 12 +++-- cpp/src/wrapper/VecImpl.h | 6 +-- cpp/src/wrapper/VecIndex.cpp | 8 ++- cpp/src/wrapper/VecIndex.h | 2 +- 12 files changed, 110 insertions(+), 26 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 1a62a791..6fdcfe65 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -49,6 +49,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-600 - Reconstruct unittest code - MS-602 - Remove zilliz namespace - MS-610 - Change error code base value from hex to decimal +- MS-635 - Add compile option to support customized faiss # Milvus 0.4.0 (2019-09-12) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index d2092eb0..cde5d3f9 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -125,6 +125,10 @@ set(MILVUS_SOURCE_DIR ${PROJECT_SOURCE_DIR}) set(MILVUS_BINARY_DIR ${PROJECT_BINARY_DIR}) set(MILVUS_ENGINE_SRC ${PROJECT_SOURCE_DIR}/src) +if (CUSTOMIZATION) + add_definitions(-DCUSTOMIZATION) +endif (CUSTOMIZATION) + include(ExternalProject) include(DefineOptions) include(BuildUtils) diff --git a/cpp/build.sh b/cpp/build.sh index 648206f9..e1525377 100755 --- a/cpp/build.sh +++ b/cpp/build.sh @@ -9,9 +9,12 @@ DB_PATH="/opt/milvus" PROFILING="OFF" USE_JFROG_CACHE="OFF" RUN_CPPLINT="OFF" +CUSTOMIZATION="ON" CUDA_COMPILER=/usr/local/cuda/bin/nvcc -while getopts "p:d:t:ulrcgjh" arg +wget -q --method HEAD + +while getopts "p:d:t:ulrcgjhx" arg do case $arg in p) @@ -45,6 +48,9 @@ do j) USE_JFROG_CACHE="ON" ;; + x) + CUSTOMIZATION="OFF" + ;; h) # help echo " @@ -87,6 +93,7 @@ CMAKE_CMD="cmake \ -DMILVUS_DB_PATH=${DB_PATH} \ -DMILVUS_ENABLE_PROFILING=${PROFILING} \ -DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \ +-DCUSTOMIZATION=${CUSTOMIZATION} \ ../" echo ${CMAKE_CMD} ${CMAKE_CMD} diff --git a/cpp/src/core/cmake/ThirdPartyPackagesCore.cmake b/cpp/src/core/cmake/ThirdPartyPackagesCore.cmake index 5691d4f9..f2823a78 100644 --- a/cpp/src/core/cmake/ThirdPartyPackagesCore.cmake +++ b/cpp/src/core/cmake/ThirdPartyPackagesCore.cmake @@ -228,17 +228,24 @@ foreach(_VERSION_ENTRY ${TOOLCHAIN_VERSIONS_TXT}) set(${_LIB_NAME} "${_LIB_VERSION}") endforeach() -if(DEFINED ENV{KNOWHERE_FAISS_URL}) - set(FAISS_SOURCE_URL "$ENV{KNOWHERE_FAISS_URL}") -else() +if(CUSTOMIZATION) set(FAISS_SOURCE_URL "http://192.168.1.105:6060/jinhai/faiss/-/archive/${FAISS_VERSION}/faiss-${FAISS_VERSION}.tar.gz") - message(STATUS "FAISS URL = ${FAISS_SOURCE_URL}") + # set(FAISS_MD5 "a589663865a8558205533c8ac414278c") + # set(FAISS_MD5 "57da9c4f599cc8fa4260488b1c96e1cc") # commit-id 6dbdf75987c34a2c853bd172ea0d384feea8358c branch-0.2.0 + # set(FAISS_MD5 "21deb1c708490ca40ecb899122c01403") # commit-id 643e48f479637fd947e7b93fa4ca72b38ecc9a39 branch-0.2.0 + # set(FAISS_MD5 "072db398351cca6e88f52d743bbb9fa0") # commit-id 3a2344d04744166af41ef1a74449d68a315bfe17 branch-0.2.1 + set(FAISS_MD5 "94988b7bdac4eb82a9575c702a3f2df3") # commit-id 1407526b31cad26f98ceca8dddaface8f18c4c19 branch-0.2.1 + + execute_process(COMMAND wget -q --method HEAD ${FAISS_SOURCE_URL} RESULT_VARIABLE return_code) + message(STATUS "Check the remote cache file ${FAISS_SOURCE_URL}. return code = ${return_code}") + if (NOT return_code EQUAL 0) + set(FAISS_SOURCE_URL "https://github.com/facebookresearch/faiss/archive/v1.5.3.tar.gz") + endif() +else() + set(FAISS_SOURCE_URL "https://github.com/facebookresearch/faiss/archive/v1.5.3.tar.gz") + set(FAISS_MD5 "0bc12737b23def156f6a1eb782050135") endif() -# set(FAISS_MD5 "a589663865a8558205533c8ac414278c") -# set(FAISS_MD5 "57da9c4f599cc8fa4260488b1c96e1cc") # commit-id 6dbdf75987c34a2c853bd172ea0d384feea8358c branch-0.2.0 -# set(FAISS_MD5 "21deb1c708490ca40ecb899122c01403") # commit-id 643e48f479637fd947e7b93fa4ca72b38ecc9a39 branch-0.2.0 -# set(FAISS_MD5 "072db398351cca6e88f52d743bbb9fa0") # commit-id 3a2344d04744166af41ef1a74449d68a315bfe17 branch-0.2.1 -set(FAISS_MD5 "5af237d77947ee632f169bcb36feee2b") # commit-id 2c8affd0da60354e4322fa4c0224519e7912b9c4 branch-0.2.1 +message(STATUS "FAISS URL = ${FAISS_SOURCE_URL}") if(DEFINED ENV{KNOWHERE_ARROW_URL}) set(ARROW_SOURCE_URL "$ENV{KNOWHERE_ARROW_URL}") diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp index 2612a636..783487be 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp @@ -64,17 +64,13 @@ FaissBaseIndex::LoadImpl(const BinarySet& index_binary) { void FaissBaseIndex::SealImpl() { - // TODO(linxj): enable - //#ifdef ZILLIZ_FAISS +#ifdef CUSTOMIZATION faiss::Index* index = index_.get(); auto idx = dynamic_cast(index); if (idx != nullptr) { idx->to_readonly(); } - // else { - // KNOHWERE_ERROR_MSG("Seal failed"); - //} - //#endif +#endif } } // namespace knowhere diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp index 11e50854..063dc635 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp @@ -55,10 +55,15 @@ VectorIndexPtr IVFSQ::CopyCpuToGpu(const int64_t& device_id, const Config& config) { if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(device_id)) { ResScope rs(res, device_id, false); + +#ifdef CUSTOMIZATION faiss::gpu::GpuClonerOptions option; option.allInGpu = true; auto gpu_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), device_id, index_.get(), &option); +#else + auto gpu_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), device_id, index_.get()); +#endif std::shared_ptr device_index; device_index.reset(gpu_index); diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp index c6c64fdc..5449aec8 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp @@ -25,6 +25,7 @@ namespace knowhere { +#ifdef CUSTOMIZATION IndexModelPtr IVFSQHybrid::Train(const DatasetPtr& dataset, const Config& config) { auto build_cfg = std::dynamic_pointer_cast(config); @@ -221,4 +222,57 @@ FaissIVFQuantizer::~FaissIVFQuantizer() { // else do nothing } +#else + +QuantizerPtr +IVFSQHybrid::LoadQuantizer(const Config& conf) { + return knowhere::QuantizerPtr(); +} + +void +IVFSQHybrid::SetQuantizer(const QuantizerPtr& q) { + +} + +void +IVFSQHybrid::UnsetQuantizer() { + +} + +void +IVFSQHybrid::LoadData(const knowhere::QuantizerPtr& q, const Config& conf) { + +} + +IndexModelPtr +IVFSQHybrid::Train(const DatasetPtr& dataset, const Config& config) { + return GPUIVFSQ::Train(dataset, config); +} + +VectorIndexPtr +IVFSQHybrid::CopyGpuToCpu(const Config& config) { + return GPUIVFSQ::CopyGpuToCpu(config); +} + +VectorIndexPtr +IVFSQHybrid::CopyCpuToGpu(const int64_t& device_id, const Config& config) { + return IVF::CopyCpuToGpu(device_id, config); +} + +void +IVFSQHybrid::search_impl(int64_t n, + const float* data, + int64_t k, + float* distances, + int64_t* labels, + const Config& cfg) { + GPUIVF::search_impl(n, data, k, distances, labels, cfg); +} + +void +IVFSQHybrid::LoadImpl(const BinarySet& index_binary) { + GPUIVF::LoadImpl(index_binary); +} + +#endif } // namespace knowhere diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.h b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.h index 73c44209..e2ca208d 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.h @@ -25,12 +25,14 @@ namespace knowhere { +#ifdef CUSTOMIZATION struct FaissIVFQuantizer : public Quantizer { faiss::gpu::GpuIndexFlat* quantizer = nullptr; ~FaissIVFQuantizer() override; }; using FaissIVFQuantizerPtr = std::shared_ptr; +#endif class IVFSQHybrid : public GPUIVFSQ { public: diff --git a/cpp/src/core/unittest/test_ivf.cpp b/cpp/src/core/unittest/test_ivf.cpp index 4e309c38..cc4712c2 100644 --- a/cpp/src/core/unittest/test_ivf.cpp +++ b/cpp/src/core/unittest/test_ivf.cpp @@ -70,7 +70,6 @@ enum class ParameterType { ivf, ivfpq, ivfsq, - ivfsqhybrid, nsg, }; @@ -104,7 +103,7 @@ class ParamGenerator { tempconf->nbits = 8; tempconf->metric_type = knowhere::METRICTYPE::L2; return tempconf; - } else if (type == ParameterType::ivfsq || type == ParameterType::ivfsqhybrid) { + } else if (type == ParameterType::ivfsq) { auto tempconf = std::make_shared(); tempconf->d = DIM; tempconf->gpu_id = device_id; @@ -158,8 +157,11 @@ INSTANTIATE_TEST_CASE_P(IVFParameters, IVFTest, // std::make_tuple("IVFPQ", ParameterType::ivfpq), // std::make_tuple("GPUIVFPQ", ParameterType::ivfpq), std::make_tuple("IVFSQ", ParameterType::ivfsq), - std::make_tuple("GPUIVFSQ", ParameterType::ivfsq), - std::make_tuple("IVFSQHybrid", ParameterType::ivfsqhybrid))); +#ifdef CUSTOMIZATION + std::make_tuple("IVFSQHybrid", ParameterType::ivfsq), +#endif + std::make_tuple("GPUIVFSQ", ParameterType::ivfsq)) + ); void AssertAnns(const knowhere::DatasetPtr& result, const int& nq, const int& k) { @@ -558,6 +560,7 @@ TEST_F(GPURESTEST, gpu_ivf_resource_test) { } } +#ifdef CUSTOMIZATION TEST_F(GPURESTEST, gpuivfsq) { { // knowhere gpu ivfsq @@ -629,6 +632,7 @@ TEST_F(GPURESTEST, gpuivfsq) { delete search_idx; } } +#endif TEST_F(GPURESTEST, copyandsearch) { // search and copy at the same time diff --git a/cpp/src/wrapper/VecImpl.h b/cpp/src/wrapper/VecImpl.h index 7558201e..fd9bb79c 100644 --- a/cpp/src/wrapper/VecImpl.h +++ b/cpp/src/wrapper/VecImpl.h @@ -17,12 +17,12 @@ #pragma once -#include "VecIndex.h" -#include "knowhere/index/vector_index/VectorIndex.h" - #include #include +#include "VecIndex.h" +#include "knowhere/index/vector_index/VectorIndex.h" + namespace milvus { namespace engine { diff --git a/cpp/src/wrapper/VecIndex.cpp b/cpp/src/wrapper/VecIndex.cpp index fe75cbb8..bd0e8724 100644 --- a/cpp/src/wrapper/VecIndex.cpp +++ b/cpp/src/wrapper/VecIndex.cpp @@ -15,8 +15,6 @@ // specific language governing permissions and limitations // under the License. -#include "wrapper/VecIndex.h" -#include "VecImpl.h" #include "knowhere/common/Exception.h" #include "knowhere/index/vector_index/IndexGPUIVF.h" #include "knowhere/index/vector_index/IndexGPUIVFPQ.h" @@ -28,6 +26,8 @@ #include "knowhere/index/vector_index/IndexIVFSQHybrid.h" #include "knowhere/index/vector_index/IndexKDT.h" #include "knowhere/index/vector_index/IndexNSG.h" +#include "wrapper/VecIndex.h" +#include "VecImpl.h" #include "utils/Log.h" #include @@ -143,10 +143,12 @@ GetVecIndexFactory(const IndexType& type, const Config& cfg) { index = std::make_shared(gpu_device); break; } +#ifdef CUSTOMIZATION case IndexType::FAISS_IVFSQ8_HYBRID: { index = std::make_shared(gpu_device); return std::make_shared(index, IndexType::FAISS_IVFSQ8_HYBRID); } +#endif case IndexType::NSG_MIX: { // TODO(linxj): bug. index = std::make_shared(gpu_device); break; @@ -159,6 +161,8 @@ GetVecIndexFactory(const IndexType& type, const Config& cfg) { VecIndexPtr LoadVecIndex(const IndexType& index_type, const knowhere::BinarySet& index_binary) { auto index = GetVecIndexFactory(index_type); + if (index == nullptr) return nullptr; + // else index->Load(index_binary); return index; } diff --git a/cpp/src/wrapper/VecIndex.h b/cpp/src/wrapper/VecIndex.h index 34d3d2f7..2284c442 100644 --- a/cpp/src/wrapper/VecIndex.h +++ b/cpp/src/wrapper/VecIndex.h @@ -20,10 +20,10 @@ #include #include -#include "cache/DataObj.h" #include "knowhere/common/BinarySet.h" #include "knowhere/common/Config.h" #include "knowhere/index/vector_index/Quantizer.h" +#include "cache/DataObj.h" #include "utils/Status.h" namespace milvus { -- GitLab