From 3426f1c954004eae40facdb49f0d5c488828c569 Mon Sep 17 00:00:00 2001 From: "xiaojun.lin" Date: Fri, 27 Sep 2019 20:27:59 +0800 Subject: [PATCH] MS-596 enable knowhere and fix compile error Former-commit-id: fee988bc22de9ac8adeb29342d0e7e9c37278304 --- cpp/src/core/CMakeLists.txt | 6 ++- .../knowhere/index/vector_index/nsg/NSG.cpp | 6 ++- cpp/src/core/test/CMakeLists.txt | 14 ++--- cpp/src/core/test/test_nsg/CMakeLists.txt | 26 +++------- cpp/src/core/test/test_nsg/test_nsg.cpp | 52 +++++++++++-------- cpp/src/wrapper/ConfAdapter.cpp | 4 ++ cpp/src/wrapper/ConfAdapter.h | 3 ++ cpp/src/wrapper/VecIndex.cpp | 2 +- cpp/unittest/wrapper/wrapper_test.cpp | 2 +- 9 files changed, 56 insertions(+), 59 deletions(-) diff --git a/cpp/src/core/CMakeLists.txt b/cpp/src/core/CMakeLists.txt index bef3f5e2..f1a13b45 100644 --- a/cpp/src/core/CMakeLists.txt +++ b/cpp/src/core/CMakeLists.txt @@ -46,9 +46,11 @@ if(NOT CMAKE_BUILD_TYPE) endif(NOT CMAKE_BUILD_TYPE) if(CMAKE_BUILD_TYPE STREQUAL "Release") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -fopenmp") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp") + set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O3") else() - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -fopenmp") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -DELPP_THREAD_SAFE -fopenmp") + set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g") endif() MESSAGE(STATUS "CMAKE_CXX_FLAGS" ${CMAKE_CXX_FLAGS}) diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.cpp index c2b5122d..0f3bb639 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.cpp @@ -24,6 +24,7 @@ #include "NSG.h" #include "knowhere/common/Exception.h" +#include "knowhere/common/Log.h" #include "knowhere/common/Timer.h" #include "NSGHelper.h" @@ -83,8 +84,9 @@ void NsgIndex::Build_with_ids(size_t nb, const float *data, const long *ids, con for (int i = 0; i < ntotal; ++i) { total_degree += nsg[i].size(); } - std::cout << "graph physical size: " << total_degree * sizeof(node_t) / 1024 / 1024; - std::cout << "average degree: " << total_degree / ntotal; + + KNOWHERE_LOG_DEBUG << "Graph physical size: " << total_degree * sizeof(node_t) / 1024 / 1024 << "m"; + KNOWHERE_LOG_DEBUG << "Average degree: " << total_degree / ntotal; ///// is_trained = true; diff --git a/cpp/src/core/test/CMakeLists.txt b/cpp/src/core/test/CMakeLists.txt index cc6ea5bd..5ad1c9e0 100644 --- a/cpp/src/core/test/CMakeLists.txt +++ b/cpp/src/core/test/CMakeLists.txt @@ -29,7 +29,7 @@ set(util_srcs ${CORE_SOURCE_DIR}/knowhere/knowhere/adapter/ArrowAdapter.cpp ${CORE_SOURCE_DIR}/knowhere/knowhere/common/Exception.cpp ${CORE_SOURCE_DIR}/knowhere/knowhere/common/Timer.cpp - utils.cpp + ${CORE_SOURCE_DIR}/test/utils.cpp ) # @@ -52,18 +52,10 @@ target_link_libraries(test_ivf ${depend_libs} ${unittest_libs} ${basic_libs}) # set(idmap_srcs - ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp - ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp - ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp - ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexIVF.cpp - ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp - ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp - ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp - ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.cpp ) if(NOT TARGET test_idmap) - add_executable(test_idmap test_idmap.cpp ${idmap_srcs} ${util_srcs}) + add_executable(test_idmap test_idmap.cpp ${idmap_srcs} ${ivf_srcs} ${util_srcs}) endif() target_link_libraries(test_idmap ${depend_libs} ${unittest_libs} ${basic_libs}) @@ -86,5 +78,5 @@ install(TARGETS test_idmap DESTINATION unittest) install(TARGETS test_kdt DESTINATION unittest) #add_subdirectory(faiss_ori) -#add_subdirectory(test_nsg) +add_subdirectory(test_nsg) diff --git a/cpp/src/core/test/test_nsg/CMakeLists.txt b/cpp/src/core/test/test_nsg/CMakeLists.txt index 17b62cd4..393cc188 100644 --- a/cpp/src/core/test/test_nsg/CMakeLists.txt +++ b/cpp/src/core/test/test_nsg/CMakeLists.txt @@ -1,6 +1,6 @@ ############################## -include_directories(/usr/local/include/gperftools) -link_directories(/usr/local/lib) +#include_directories(/usr/local/include/gperftools) +#link_directories(/usr/local/lib) add_definitions(-std=c++11 -O3 -lboost -march=native -Wall -DINFO) @@ -13,29 +13,15 @@ else () endif () message(${OpenMP_CXX_FLAGS}) -include_directories(${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg) - -aux_source_directory(${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg nsg_src) +include_directories(${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/nsg) +aux_source_directory(${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/nsg nsg_src) set(interface_src - ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp - ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp - ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp - ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp - ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg_index.cpp - ${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp - ${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp - ${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp - ../utils.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexNSG.cpp ) if(NOT TARGET test_nsg) - add_executable(test_nsg - test_nsg.cpp - ${interface_src} - ${nsg_src} - ${util_srcs} - ) + add_executable(test_nsg test_nsg.cpp ${interface_src} ${nsg_src} ${util_srcs} ${ivf_srcs}) endif() target_link_libraries(test_nsg ${depend_libs} ${unittest_libs} ${basic_libs}) diff --git a/cpp/src/core/test/test_nsg/test_nsg.cpp b/cpp/src/core/test/test_nsg/test_nsg.cpp index b1b1edef..8f0187e0 100644 --- a/cpp/src/core/test/test_nsg/test_nsg.cpp +++ b/cpp/src/core/test/test_nsg/test_nsg.cpp @@ -19,10 +19,11 @@ #include #include -#include "knowhere/common/exception.h" -#include "knowhere/index/vector_index/gpu_ivf.h" -#include "knowhere/index/vector_index/nsg_index.h" -#include "knowhere/index/vector_index/nsg/nsg_io.h" +#include "knowhere/common/Exception.h" +#include "knowhere/index/vector_index/FaissBaseIndex.h" +#include "knowhere/index/vector_index/IndexNSG.h" +#include "knowhere/index/vector_index/nsg/NSGIO.h" +#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h" #include "../utils.h" @@ -32,16 +33,31 @@ using ::testing::TestWithParam; using ::testing::Values; using ::testing::Combine; -constexpr int64_t DEVICE_ID = 0; +constexpr int64_t DEVICE_ID = 1; -class NSGInterfaceTest : public DataGen, public TestWithParam<::std::tuple> { + class NSGInterfaceTest : public DataGen, public ::testing::Test { protected: void SetUp() override { //Init_with_default(); FaissGpuResourceMgr::GetInstance().InitDevice(DEVICE_ID, 1024*1024*200, 1024*1024*600, 2); - Generate(256, 10000, 1); + Generate(256, 1000000, 1); index_ = std::make_shared(); - std::tie(train_cfg, search_cfg) = GetParam(); + + auto tmp_conf = std::make_shared(); + tmp_conf->gpu_id = DEVICE_ID; + tmp_conf->knng = 100; + tmp_conf->nprobe = 32; + tmp_conf->nlist = 16384; + tmp_conf->search_length = 60; + tmp_conf->out_degree = 70; + tmp_conf->candidate_pool_size = 500; + tmp_conf->metric_type = METRICTYPE::L2; + train_conf = tmp_conf; + + auto tmp2_conf = std::make_shared(); + tmp2_conf->k = k; + tmp2_conf->search_length = 30; + search_conf = tmp2_conf; } void TearDown() override { @@ -50,18 +66,10 @@ class NSGInterfaceTest : public DataGen, public TestWithParam<::std::tuple index_; - Config train_cfg; - Config search_cfg; + Config train_conf; + Config search_conf; }; -INSTANTIATE_TEST_CASE_P(NSGparameters, NSGInterfaceTest, - Values(std::make_tuple( - // search length > out_degree - Config::object{{"nlist", 128}, {"nprobe", 50}, {"knng", 100}, {"metric_type", "L2"}, - {"search_length", 60}, {"out_degree", 70}, {"candidate_pool_size", 500}}, - Config::object{{"k", 20}, {"search_length", 30}})) -); - void AssertAnns(const DatasetPtr &result, const int &nq, const int &k) { @@ -71,17 +79,17 @@ void AssertAnns(const DatasetPtr &result, } } -TEST_P(NSGInterfaceTest, basic_test) { +TEST_F(NSGInterfaceTest, basic_test) { assert(!xb.empty()); - auto model = index_->Train(base_dataset, train_cfg); - auto result = index_->Search(query_dataset, search_cfg); + auto model = index_->Train(base_dataset, train_conf); + auto result = index_->Search(query_dataset, search_conf); AssertAnns(result, nq, k); auto binaryset = index_->Serialize(); auto new_index = std::make_shared(); new_index->Load(binaryset); - auto new_result = new_index->Search(query_dataset, Config::object{{"k", k}}); + auto new_result = new_index->Search(query_dataset, search_conf); AssertAnns(result, nq, k); ASSERT_EQ(index_->Count(), nb); diff --git a/cpp/src/wrapper/ConfAdapter.cpp b/cpp/src/wrapper/ConfAdapter.cpp index ea6be1ac..1ccbaee9 100644 --- a/cpp/src/wrapper/ConfAdapter.cpp +++ b/cpp/src/wrapper/ConfAdapter.cpp @@ -145,6 +145,10 @@ NSGConfAdapter::Match(const TempMetaConf &metaconf) { conf->out_degree = 50 + 5 * scale_factor; conf->candidate_pool_size = 200 + 100 * scale_factor; MatchBase(conf); + +// WRAPPER_LOG_DEBUG << "nlist: " << conf->nlist +// << ", gpu_id: " << conf->gpu_id << ", d: " << conf->d +// << ", nprobe: " << conf->nprobe << ", knng: " << conf->knng; return conf; } diff --git a/cpp/src/wrapper/ConfAdapter.h b/cpp/src/wrapper/ConfAdapter.h index d0f032cb..d8281d88 100644 --- a/cpp/src/wrapper/ConfAdapter.h +++ b/cpp/src/wrapper/ConfAdapter.h @@ -47,6 +47,9 @@ class ConfAdapter { virtual knowhere::Config MatchSearch(const TempMetaConf &metaconf, const IndexType &type); +// virtual void +// Dump(){} + protected: static void MatchBase(knowhere::Config conf); }; diff --git a/cpp/src/wrapper/VecIndex.cpp b/cpp/src/wrapper/VecIndex.cpp index 734f6633..ffc14ebd 100644 --- a/cpp/src/wrapper/VecIndex.cpp +++ b/cpp/src/wrapper/VecIndex.cpp @@ -137,7 +137,7 @@ GetVecIndexFactory(const IndexType &type, const Config &cfg) { index = std::make_shared(gpu_device); break; } - case IndexType::NSG_MIX: { // TODO(linxj): bug. + case IndexType::NSG_MIX: { index = std::make_shared(gpu_device); break; } diff --git a/cpp/unittest/wrapper/wrapper_test.cpp b/cpp/unittest/wrapper/wrapper_test.cpp index d19f9807..43c4103a 100644 --- a/cpp/unittest/wrapper/wrapper_test.cpp +++ b/cpp/unittest/wrapper/wrapper_test.cpp @@ -173,7 +173,7 @@ INSTANTIATE_TEST_CASE_P(WrapperParam, KnowhereWrapperTest, std::make_tuple(IndexType::FAISS_IVFSQ8_CPU, "Default", DIM, NB, 10, 10), std::make_tuple(IndexType::FAISS_IVFSQ8_GPU, "Default", DIM, NB, 10, 10), std::make_tuple(IndexType::FAISS_IVFSQ8_MIX, "Default", DIM, NB, 10, 10), -// std::make_tuple(IndexType::NSG_MIX, "Default", 128, 250000, 10, 10), + std::make_tuple(IndexType::NSG_MIX, "Default", 128, 250000, 10, 10), // std::make_tuple(IndexType::SPTAG_KDT_RNT_CPU, "Default", 128, 250000, 10, 10), std::make_tuple(IndexType::FAISS_IDMAP, "Default", 64, 100000, 10, 10) ) -- GitLab