From 993558a00eb36518b26224fffe1ff22d6d50efcb Mon Sep 17 00:00:00 2001 From: "xiaojun.lin" Date: Fri, 27 Sep 2019 15:41:20 +0800 Subject: [PATCH] MS-565 remove jsoncons v2 Former-commit-id: 64e514a130640eae43c4e9657a3fcbe2efac3074 --- cpp/src/core/knowhere/CMakeLists.txt | 3 - cpp/src/core/test/test_ivf.cpp | 4 +- cpp/src/db/engine/ExecutionEngineImpl.cpp | 51 ++++--- cpp/src/wrapper/ConfAdapter.cpp | 161 ++++++++++++++++++++++ cpp/src/wrapper/ConfAdapter.h | 91 ++++++++++++ cpp/src/wrapper/ConfAdapterMgr.cpp | 64 +++++++++ cpp/src/wrapper/ConfAdapterMgr.h | 63 +++++++++ cpp/src/wrapper/VecImpl.cpp | 35 ++--- cpp/src/wrapper/VecIndex.cpp | 71 +--------- cpp/src/wrapper/VecIndex.h | 7 - cpp/unittest/wrapper/wrapper_test.cpp | 157 ++++++++++++--------- 11 files changed, 519 insertions(+), 188 deletions(-) create mode 100644 cpp/src/wrapper/ConfAdapter.cpp create mode 100644 cpp/src/wrapper/ConfAdapter.h create mode 100644 cpp/src/wrapper/ConfAdapterMgr.cpp create mode 100644 cpp/src/wrapper/ConfAdapterMgr.h diff --git a/cpp/src/core/knowhere/CMakeLists.txt b/cpp/src/core/knowhere/CMakeLists.txt index 864d0358..4d596e24 100644 --- a/cpp/src/core/knowhere/CMakeLists.txt +++ b/cpp/src/core/knowhere/CMakeLists.txt @@ -117,7 +117,6 @@ set(CORE_INCLUDE_DIRS ${CORE_SOURCE_DIR}/knowhere ${CORE_SOURCE_DIR}/thirdparty ${CORE_SOURCE_DIR}/thirdparty/SPTAG/AnnService - ${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include ${ARROW_INCLUDE_DIR} ${FAISS_INCLUDE_DIR} ${OPENBLAS_INCLUDE_DIR} @@ -129,8 +128,6 @@ set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE) #INSTALL(DIRECTORY # ${CORE_SOURCE_DIR}/include/knowhere -# ${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons -# ${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons_ext # ${ARROW_INCLUDE_DIR}/arrow # ${FAISS_PREFIX}/include/faiss # ${OPENBLAS_INCLUDE_DIR}/ diff --git a/cpp/src/core/test/test_ivf.cpp b/cpp/src/core/test/test_ivf.cpp index 579729f2..1eeb1f98 100644 --- a/cpp/src/core/test/test_ivf.cpp +++ b/cpp/src/core/test/test_ivf.cpp @@ -146,8 +146,8 @@ INSTANTIATE_TEST_CASE_P(IVFParameters, IVFTest, Values( std::make_tuple("IVF", ParameterType::ivf), std::make_tuple("GPUIVF", ParameterType::ivf), - std::make_tuple("IVFPQ", ParameterType::ivfpq), - std::make_tuple("GPUIVFPQ", ParameterType::ivfpq), +// std::make_tuple("IVFPQ", ParameterType::ivfpq), +// std::make_tuple("GPUIVFPQ", ParameterType::ivfpq), std::make_tuple("IVFSQ", ParameterType::ivfsq), std::make_tuple("GPUIVFSQ", ParameterType::ivfsq) ) diff --git a/cpp/src/db/engine/ExecutionEngineImpl.cpp b/cpp/src/db/engine/ExecutionEngineImpl.cpp index b9d18b4a..1541cb05 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.cpp +++ b/cpp/src/db/engine/ExecutionEngineImpl.cpp @@ -26,6 +26,9 @@ #include "src/wrapper/VecIndex.h" #include "src/wrapper/VecImpl.h" #include "knowhere/common/Exception.h" +#include "knowhere/common/Config.h" +#include "wrapper/ConfAdapterMgr.h" +#include "wrapper/ConfAdapter.h" #include "server/Config.h" #include @@ -50,11 +53,15 @@ ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension, throw Exception(DB_ERROR, "Could not create VecIndex"); } - Config build_cfg; - build_cfg["dim"] = dimension; - build_cfg["metric_type"] = (metric_type_ == MetricType::IP) ? "IP" : "L2"; - AutoGenParams(index_->GetType(), 0, build_cfg); - auto ec = std::static_pointer_cast(index_)->Build(build_cfg); + TempMetaConf temp_conf; + temp_conf.gpu_id = gpu_num_; + temp_conf.dim = dimension; + temp_conf.metric_type = (metric_type_ == MetricType::IP) ? + knowhere::METRICTYPE::IP : knowhere::METRICTYPE::L2; + auto adapter = AdapterMgr::GetInstance().GetAdapter(index_->GetType()); + auto conf = adapter->Match(temp_conf); + + auto ec = std::static_pointer_cast(index_)->Build(conf); if (ec != KNOWHERE_SUCCESS) { throw Exception(DB_ERROR, "Build index error"); } @@ -274,17 +281,21 @@ ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_t throw Exception(DB_ERROR, "Could not create VecIndex"); } - Config build_cfg; - build_cfg["dim"] = Dimension(); - build_cfg["metric_type"] = (metric_type_ == MetricType::IP) ? "IP" : "L2"; - build_cfg["gpu_id"] = gpu_num_; - build_cfg["nlist"] = nlist_; - AutoGenParams(to_index->GetType(), Count(), build_cfg); + TempMetaConf temp_conf; + temp_conf.gpu_id = gpu_num_; + temp_conf.dim = Dimension(); + temp_conf.nlist = nlist_; + temp_conf.metric_type = (metric_type_ == MetricType::IP) ? + knowhere::METRICTYPE::IP : knowhere::METRICTYPE::L2; + temp_conf.size = Count(); + + auto adapter = AdapterMgr::GetInstance().GetAdapter(to_index->GetType()); + auto conf = adapter->Match(temp_conf); auto status = to_index->BuildAll(Count(), - from_index->GetRawVectors(), - from_index->GetRawIds(), - build_cfg); + from_index->GetRawVectors(), + from_index->GetRawIds(), + conf); if (!status.ok()) { throw Exception(DB_ERROR, status.message()); } return std::make_shared(to_index, location, engine_type, metric_type_, nlist_); @@ -302,8 +313,16 @@ Status ExecutionEngineImpl::Search(long n, } ENGINE_LOG_DEBUG << "Search Params: [k] " << k << " [nprobe] " << nprobe; - auto cfg = Config::object{{"k", k}, {"nprobe", nprobe}}; - auto status = index_->Search(n, data, distances, labels, cfg); + + // TODO(linxj): remove here. Get conf from function + TempMetaConf temp_conf; + temp_conf.k = k; + temp_conf.nprobe = nprobe; + + auto adapter = AdapterMgr::GetInstance().GetAdapter(index_->GetType()); + auto conf = adapter->MatchSearch(temp_conf, index_->GetType()); + + auto status = index_->Search(n, data, distances, labels, conf); if (!status.ok()) { ENGINE_LOG_ERROR << "Search error"; } diff --git a/cpp/src/wrapper/ConfAdapter.cpp b/cpp/src/wrapper/ConfAdapter.cpp new file mode 100644 index 00000000..ea6be1ac --- /dev/null +++ b/cpp/src/wrapper/ConfAdapter.cpp @@ -0,0 +1,161 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#include +#include "ConfAdapter.h" +#include "src/utils/Log.h" +#include "knowhere/index/vector_index/helpers/IndexParameter.h" + +// TODO: add conf checker + +namespace zilliz { +namespace milvus { +namespace engine { + +#if CUDA_VERSION > 9000 +#define GPU_MAX_NRPOBE 2048 +#else +#define GPU_MAX_NRPOBE 1024 +#endif + +void +ConfAdapter::MatchBase(knowhere::Config conf) { + if (conf->metric_type == knowhere::DEFAULT_TYPE) + conf->metric_type = knowhere::METRICTYPE::L2; + if (conf->gpu_id == knowhere::INVALID_VALUE) + conf->gpu_id = 0; +} + +knowhere::Config +ConfAdapter::Match(const TempMetaConf &metaconf) { + auto conf = std::make_shared(); + conf->d = metaconf.dim; + conf->metric_type = metaconf.metric_type; + conf->gpu_id = conf->gpu_id; + MatchBase(conf); + return conf; +} + +knowhere::Config +ConfAdapter::MatchSearch(const TempMetaConf &metaconf, const IndexType &type) { + auto conf = std::make_shared(); + conf->k = metaconf.k; + return conf; +} + +knowhere::Config +IVFConfAdapter::Match(const TempMetaConf &metaconf) { + auto conf = std::make_shared(); + conf->nlist = MatchNlist(metaconf.size, metaconf.nlist); + conf->d = metaconf.dim; + conf->metric_type = metaconf.metric_type; + conf->gpu_id = conf->gpu_id; + MatchBase(conf); + return conf; +} + +static constexpr float TYPICAL_COUNT = 1000000.0; + +int64_t +IVFConfAdapter::MatchNlist(const int64_t &size, const int64_t &nlist) { + if (size <= TYPICAL_COUNT / 16384 + 1) { + // handle less row count, avoid nlist set to 0 + return 1; + } else if (int(size / TYPICAL_COUNT) * nlist == 0) { + // calculate a proper nlist if nlist not specified or size less than TYPICAL_COUNT + return int(size / TYPICAL_COUNT * 16384); + } + return 0; +} + +knowhere::Config +IVFConfAdapter::MatchSearch(const TempMetaConf &metaconf, const IndexType &type) { + auto conf = std::make_shared(); + conf->k = metaconf.k; + conf->nprobe = metaconf.nprobe; + + switch (type) { + case IndexType::FAISS_IVFFLAT_GPU: + case IndexType::FAISS_IVFSQ8_GPU: + case IndexType::FAISS_IVFPQ_GPU: + if (conf->nprobe > GPU_MAX_NRPOBE) { + WRAPPER_LOG_WARNING << "When search with GPU, nprobe shoud be no more than " << GPU_MAX_NRPOBE + << ", but you passed " << conf->nprobe + << ". Search with " << GPU_MAX_NRPOBE << " instead"; + conf->nprobe = GPU_MAX_NRPOBE; + + } + } + return conf; +} + +knowhere::Config +IVFSQConfAdapter::Match(const TempMetaConf &metaconf) { + auto conf = std::make_shared(); + conf->nlist = MatchNlist(metaconf.size, metaconf.nlist); + conf->d = metaconf.dim; + conf->metric_type = metaconf.metric_type; + conf->gpu_id = conf->gpu_id; + conf->nbits = 8; + MatchBase(conf); + return conf; +} + +knowhere::Config +IVFPQConfAdapter::Match(const TempMetaConf &metaconf) { + auto conf = std::make_shared(); + conf->nlist = MatchNlist(metaconf.size, metaconf.nlist); + conf->d = metaconf.dim; + conf->metric_type = metaconf.metric_type; + conf->gpu_id = conf->gpu_id; + conf->nbits = 8; + conf->m = 8; + MatchBase(conf); + return conf; +} + +knowhere::Config +NSGConfAdapter::Match(const TempMetaConf &metaconf) { + auto conf = std::make_shared(); + conf->nlist = MatchNlist(metaconf.size, metaconf.nlist); + conf->d = metaconf.dim; + conf->metric_type = metaconf.metric_type; + conf->gpu_id = conf->gpu_id; + + auto scale_factor = round(metaconf.dim / 128.0); + scale_factor = scale_factor >= 4 ? 4 : scale_factor; + conf->nprobe = 6 + 10 * scale_factor; + conf->knng = 100 + 100 * scale_factor; + conf->search_length = 40 + 5 * scale_factor; + conf->out_degree = 50 + 5 * scale_factor; + conf->candidate_pool_size = 200 + 100 * scale_factor; + MatchBase(conf); + return conf; +} + +knowhere::Config +NSGConfAdapter::MatchSearch(const TempMetaConf &metaconf, const IndexType &type) { + auto conf = std::make_shared(); + conf->k = metaconf.k; + conf->search_length = metaconf.search_length; + return conf; +} + +} +} +} diff --git a/cpp/src/wrapper/ConfAdapter.h b/cpp/src/wrapper/ConfAdapter.h new file mode 100644 index 00000000..d0f032cb --- /dev/null +++ b/cpp/src/wrapper/ConfAdapter.h @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include "knowhere/common/Config.h" +#include "VecIndex.h" + + +namespace zilliz { +namespace milvus { +namespace engine { + +// TODO(linxj): remove later, replace with real metaconf +constexpr int64_t TEMPMETA_DEFAULT_VALUE = -1; +struct TempMetaConf { + int64_t size = TEMPMETA_DEFAULT_VALUE; + int64_t nlist = TEMPMETA_DEFAULT_VALUE; + int64_t dim = TEMPMETA_DEFAULT_VALUE; + int64_t gpu_id = TEMPMETA_DEFAULT_VALUE; + int64_t k = TEMPMETA_DEFAULT_VALUE; + int64_t nprobe = TEMPMETA_DEFAULT_VALUE; + int64_t search_length = TEMPMETA_DEFAULT_VALUE; + knowhere::METRICTYPE metric_type = knowhere::DEFAULT_TYPE; +}; + +class ConfAdapter { + public: + virtual knowhere::Config + Match(const TempMetaConf &metaconf); + + virtual knowhere::Config + MatchSearch(const TempMetaConf &metaconf, const IndexType &type); + + protected: + static void MatchBase(knowhere::Config conf); +}; + +using ConfAdapterPtr = std::shared_ptr; + +class IVFConfAdapter : public ConfAdapter { + public: + knowhere::Config + Match(const TempMetaConf &metaconf) override; + + knowhere::Config + MatchSearch(const TempMetaConf &metaconf, const IndexType &type) override; + + protected: + static int64_t MatchNlist(const int64_t &size, const int64_t &nlist); +}; + +class IVFSQConfAdapter : public IVFConfAdapter { + public: + knowhere::Config + Match(const TempMetaConf &metaconf) override; +}; + +class IVFPQConfAdapter : public IVFConfAdapter { + public: + knowhere::Config + Match(const TempMetaConf &metaconf) override; +}; + +class NSGConfAdapter : public IVFConfAdapter { + public: + knowhere::Config + Match(const TempMetaConf &metaconf) override; + + knowhere::Config + MatchSearch(const TempMetaConf &metaconf, const IndexType &type) final; +}; + +} +} +} diff --git a/cpp/src/wrapper/ConfAdapterMgr.cpp b/cpp/src/wrapper/ConfAdapterMgr.cpp new file mode 100644 index 00000000..cb7331a9 --- /dev/null +++ b/cpp/src/wrapper/ConfAdapterMgr.cpp @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#include "src/utils/Exception.h" +#include "ConfAdapterMgr.h" + + +namespace zilliz { +namespace milvus { +namespace engine { + +ConfAdapterPtr +AdapterMgr::GetAdapter(const IndexType &indexType) { + if (!init_) RegisterAdapter(); + + auto it = table_.find(indexType); + if (it != table_.end()) { + return it->second(); + } else { + throw Exception(KNOWHERE_INVALID_ARGUMENT, "Can not find this type of confadapter"); + } +} + + +#define REGISTER_CONF_ADAPTER(T, KEY, NAME) static AdapterMgr::register_treg_##NAME##_(KEY) +void +AdapterMgr::RegisterAdapter() { + init_ = true; + + REGISTER_CONF_ADAPTER(ConfAdapter, IndexType::FAISS_IDMAP, idmap); + + REGISTER_CONF_ADAPTER(IVFConfAdapter, IndexType::FAISS_IVFFLAT_CPU, ivf_cpu); + REGISTER_CONF_ADAPTER(IVFConfAdapter, IndexType::FAISS_IVFFLAT_GPU, ivf_gpu); + REGISTER_CONF_ADAPTER(IVFConfAdapter, IndexType::FAISS_IVFFLAT_MIX, ivf_mix); + + REGISTER_CONF_ADAPTER(IVFSQConfAdapter, IndexType::FAISS_IVFSQ8_CPU, ivfsq8_cpu); + REGISTER_CONF_ADAPTER(IVFSQConfAdapter, IndexType::FAISS_IVFSQ8_GPU, ivfsq8_gpu); + REGISTER_CONF_ADAPTER(IVFSQConfAdapter, IndexType::FAISS_IVFSQ8_MIX, ivfsq8_mix); + + REGISTER_CONF_ADAPTER(IVFPQConfAdapter, IndexType::FAISS_IVFPQ_CPU, ivfpq_cpu); + REGISTER_CONF_ADAPTER(IVFPQConfAdapter, IndexType::FAISS_IVFPQ_GPU, ivfpq_gpu); + + REGISTER_CONF_ADAPTER(NSGConfAdapter, IndexType::NSG_MIX, nsg_mix); +} + +} // engine +} // milvus +} // zilliz + diff --git a/cpp/src/wrapper/ConfAdapterMgr.h b/cpp/src/wrapper/ConfAdapterMgr.h new file mode 100644 index 00000000..fe8b4f07 --- /dev/null +++ b/cpp/src/wrapper/ConfAdapterMgr.h @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include "VecIndex.h" +#include "ConfAdapter.h" + + +namespace zilliz { +namespace milvus { +namespace engine { + +class AdapterMgr { + public: + template + struct register_t { + explicit register_t(const IndexType &key) { + AdapterMgr::GetInstance().table_.emplace(key, [] { + return std::make_shared(); + }); + } + }; + + static AdapterMgr & + GetInstance() { + static AdapterMgr instance; + return instance; + } + + ConfAdapterPtr + GetAdapter(const IndexType &indexType); + + void + RegisterAdapter(); + + protected: + bool init_ = false; + std::map > table_; +}; + + +} // engine +} // milvus +} // zilliz + + + diff --git a/cpp/src/wrapper/VecImpl.cpp b/cpp/src/wrapper/VecImpl.cpp index 24f6a2bb..e536fa88 100644 --- a/cpp/src/wrapper/VecImpl.cpp +++ b/cpp/src/wrapper/VecImpl.cpp @@ -24,6 +24,10 @@ #include "VecImpl.h" #include "DataTransfer.h" +/* + * no parameter check in this layer. + * only responible for index combination + */ namespace zilliz { namespace milvus { @@ -39,7 +43,7 @@ VecIndexImpl::BuildAll(const long &nb, const long &nt, const float *xt) { try { - dim = cfg["dim"].as(); + dim = cfg->d; auto dataset = GenDatasetWithIds(nb, dim, xb, ids); auto preprocessor = index_->BuildPreprocessor(dataset, cfg); @@ -50,9 +54,6 @@ VecIndexImpl::BuildAll(const long &nb, } catch (KnowhereException &e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); - } catch (jsoncons::json_exception &e) { - WRAPPER_LOG_ERROR << e.what(); - return Status(KNOWHERE_INVALID_ARGUMENT, e.what()); } catch (std::exception &e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_ERROR, e.what()); @@ -69,9 +70,6 @@ VecIndexImpl::Add(const long &nb, const float *xb, const long *ids, const Config } catch (KnowhereException &e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); - } catch (jsoncons::json_exception &e) { - WRAPPER_LOG_ERROR << e.what(); - return Status(KNOWHERE_INVALID_ARGUMENT, e.what()); } catch (std::exception &e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_ERROR, e.what()); @@ -82,13 +80,11 @@ VecIndexImpl::Add(const long &nb, const float *xb, const long *ids, const Config Status VecIndexImpl::Search(const long &nq, const float *xq, float *dist, long *ids, const Config &cfg) { try { - auto k = cfg["k"].as(); + auto k = cfg->k; auto dataset = GenDataset(nq, dim, xq); Config search_cfg = cfg; - ParameterValidation(type, search_cfg); - auto res = index_->Search(dataset, search_cfg); auto ids_array = res->array()[0]; auto dis_array = res->array()[1]; @@ -120,9 +116,6 @@ VecIndexImpl::Search(const long &nq, const float *xq, float *dist, long *ids, co } catch (KnowhereException &e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); - } catch (jsoncons::json_exception &e) { - WRAPPER_LOG_ERROR << e.what(); - return Status(KNOWHERE_INVALID_ARGUMENT, e.what()); } catch (std::exception &e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_ERROR, e.what()); @@ -208,14 +201,11 @@ BFIndex::GetRawIds() { ErrorCode BFIndex::Build(const Config &cfg) { try { - dim = cfg["dim"].as(); + dim = cfg->d; std::static_pointer_cast(index_)->Train(cfg); } catch (KnowhereException &e) { WRAPPER_LOG_ERROR << e.what(); return KNOWHERE_UNEXPECTED_ERROR; - } catch (jsoncons::json_exception &e) { - WRAPPER_LOG_ERROR << e.what(); - return KNOWHERE_INVALID_ARGUMENT; } catch (std::exception &e) { WRAPPER_LOG_ERROR << e.what(); return KNOWHERE_ERROR; @@ -231,7 +221,7 @@ BFIndex::BuildAll(const long &nb, const long &nt, const float *xt) { try { - dim = cfg["dim"].as(); + dim = cfg->d; auto dataset = GenDatasetWithIds(nb, dim, xb, ids); std::static_pointer_cast(index_)->Train(cfg); @@ -239,9 +229,6 @@ BFIndex::BuildAll(const long &nb, } catch (KnowhereException &e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); - } catch (jsoncons::json_exception &e) { - WRAPPER_LOG_ERROR << e.what(); - return Status(KNOWHERE_INVALID_ARGUMENT, e.what()); } catch (std::exception &e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_ERROR, e.what()); @@ -258,7 +245,7 @@ IVFMixIndex::BuildAll(const long &nb, const long &nt, const float *xt) { try { - dim = cfg["dim"].as(); + dim = cfg->d; auto dataset = GenDatasetWithIds(nb, dim, xb, ids); auto preprocessor = index_->BuildPreprocessor(dataset, cfg); @@ -278,9 +265,6 @@ IVFMixIndex::BuildAll(const long &nb, } catch (KnowhereException &e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); - } catch (jsoncons::json_exception &e) { - WRAPPER_LOG_ERROR << e.what(); - return Status(KNOWHERE_INVALID_ARGUMENT, e.what()); } catch (std::exception &e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_ERROR, e.what()); @@ -290,7 +274,6 @@ IVFMixIndex::BuildAll(const long &nb, Status IVFMixIndex::Load(const zilliz::knowhere::BinarySet &index_binary) { - //index_ = std::make_shared(); index_->Load(index_binary); dim = Dimension(); return Status::OK(); diff --git a/cpp/src/wrapper/VecIndex.cpp b/cpp/src/wrapper/VecIndex.cpp index cd8843e5..71d4ab96 100644 --- a/cpp/src/wrapper/VecIndex.cpp +++ b/cpp/src/wrapper/VecIndex.cpp @@ -36,8 +36,6 @@ namespace zilliz { namespace milvus { namespace engine { -static constexpr float TYPICAL_COUNT = 1000000.0; - struct FileIOReader { std::fstream fs; std::string name; @@ -101,7 +99,7 @@ FileIOWriter::operator()(void *ptr, size_t size) { VecIndexPtr GetVecIndexFactory(const IndexType &type, const Config &cfg) { std::shared_ptr index; - auto gpu_device = cfg.get_with_default("gpu_id", 0); + auto gpu_device = -1; // TODO(linxj): remove hardcode here switch (type) { case IndexType::FAISS_IDMAP: { index = std::make_shared(); @@ -112,12 +110,11 @@ GetVecIndexFactory(const IndexType &type, const Config &cfg) { break; } case IndexType::FAISS_IVFFLAT_GPU: { - // TODO(linxj): 规范化参数 index = std::make_shared(gpu_device); break; } case IndexType::FAISS_IVFFLAT_MIX: { - index = std::make_shared(0); + index = std::make_shared(gpu_device); return std::make_shared(index, IndexType::FAISS_IVFFLAT_MIX); } case IndexType::FAISS_IVFPQ_CPU: { @@ -242,70 +239,6 @@ write_index(VecIndexPtr index, const std::string &location) { return Status::OK(); } - -// TODO(linxj): redo here. -void -AutoGenParams(const IndexType &type, const long &size, zilliz::knowhere::Config &cfg) { - auto nlist = cfg.get_with_default("nlist", 0); - if (size <= TYPICAL_COUNT / 16384 + 1) { - //handle less row count, avoid nlist set to 0 - cfg["nlist"] = 1; - } else if (int(size / TYPICAL_COUNT) * nlist == 0) { - //calculate a proper nlist if nlist not specified or size less than TYPICAL_COUNT - cfg["nlist"] = int(size / TYPICAL_COUNT * 16384); - } - - if (!cfg.contains("gpu_id")) { cfg["gpu_id"] = int(0); } - if (!cfg.contains("metric_type")) { cfg["metric_type"] = "L2"; } - - switch (type) { - case IndexType::FAISS_IVFSQ8_MIX: { - if (!cfg.contains("nbits")) { cfg["nbits"] = int(8); } - break; - } - case IndexType::NSG_MIX: { - auto scale_factor = round(cfg["dim"].as() / 128.0); - scale_factor = scale_factor >= 4 ? 4 : scale_factor; - cfg["nlist"] = int(size / 1000000.0 * 8192); - if (!cfg.contains("nprobe")) { cfg["nprobe"] = 6 + 10 * scale_factor; } - if (!cfg.contains("knng")) { cfg["knng"] = 100 + 100 * scale_factor; } - if (!cfg.contains("search_length")) { cfg["search_length"] = 40 + 5 * scale_factor; } - if (!cfg.contains("out_degree")) { cfg["out_degree"] = 50 + 5 * scale_factor; } - if (!cfg.contains("candidate_pool_size")) { cfg["candidate_pool_size"] = 200 + 100 * scale_factor; } - WRAPPER_LOG_DEBUG << pretty_print(cfg); - break; - } - } -} - -#if CUDA_VERSION > 9000 -#define GPU_MAX_NRPOBE 2048 -#else -#define GPU_MAX_NRPOBE 1024 -#endif - -void -ParameterValidation(const IndexType &type, Config &cfg) { - switch (type) { - case IndexType::FAISS_IVFSQ8_GPU: - case IndexType::FAISS_IVFFLAT_GPU: - case IndexType::FAISS_IVFPQ_GPU: { - //search on GPU - if (cfg.get_with_default("nprobe", 0) != 0) { - auto nprobe = cfg["nprobe"].as(); - if (nprobe > GPU_MAX_NRPOBE) { - WRAPPER_LOG_WARNING << "When search with GPU, nprobe shoud be no more than " << GPU_MAX_NRPOBE - << ", but you passed " << nprobe - << ". Search with " << GPU_MAX_NRPOBE << " instead"; - cfg.insert_or_assign("nprobe", GPU_MAX_NRPOBE); - } - } - break; - } - default:break; - } -} - IndexType ConvertToCpuIndexType(const IndexType &type) { // TODO(linxj): add IDMAP diff --git a/cpp/src/wrapper/VecIndex.h b/cpp/src/wrapper/VecIndex.h index 3737270f..4ead6aaf 100644 --- a/cpp/src/wrapper/VecIndex.h +++ b/cpp/src/wrapper/VecIndex.h @@ -30,7 +30,6 @@ namespace zilliz { namespace milvus { namespace engine { -// TODO(linxj): jsoncons => rapidjson or other. using Config = zilliz::knowhere::Config; enum class IndexType { @@ -116,12 +115,6 @@ GetVecIndexFactory(const IndexType &type, const Config &cfg = Config()); extern VecIndexPtr LoadVecIndex(const IndexType &index_type, const zilliz::knowhere::BinarySet &index_binary); -extern void -AutoGenParams(const IndexType &type, const long &size, Config &cfg); - -extern void -ParameterValidation(const IndexType &type, Config &cfg); - extern IndexType ConvertToCpuIndexType(const IndexType &type); diff --git a/cpp/unittest/wrapper/wrapper_test.cpp b/cpp/unittest/wrapper/wrapper_test.cpp index 4c50201f..d19f9807 100644 --- a/cpp/unittest/wrapper/wrapper_test.cpp +++ b/cpp/unittest/wrapper/wrapper_test.cpp @@ -18,10 +18,12 @@ #include "utils/easylogging++.h" #include "src/wrapper/VecIndex.h" #include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h" +#include "knowhere/index/vector_index/helpers/IndexParameter.h" #include "utils.h" #include + INITIALIZE_EASYLOGGINGPP using namespace zilliz::milvus::engine; @@ -35,19 +37,83 @@ constexpr int64_t DIM = 128; constexpr int64_t NB = 100000; constexpr int64_t DEVICE_ID = 0; +class ParamGenerator { + public: + static ParamGenerator& GetInstance(){ + static ParamGenerator instance; + return instance; + } + + Config Gen(const IndexType& type) { + switch (type) { + case IndexType::FAISS_IDMAP: { + auto tempconf = std::make_shared(); + tempconf->metric_type = zilliz::knowhere::METRICTYPE::L2; + return tempconf; + } + case IndexType::FAISS_IVFFLAT_CPU: + case IndexType::FAISS_IVFFLAT_GPU: + case IndexType::FAISS_IVFFLAT_MIX: { + auto tempconf = std::make_shared(); + tempconf->nlist = 100; + tempconf->nprobe = 16; + tempconf->metric_type = zilliz::knowhere::METRICTYPE::L2; + return tempconf; + } + case IndexType::FAISS_IVFSQ8_CPU: + case IndexType::FAISS_IVFSQ8_GPU: + case IndexType::FAISS_IVFSQ8_MIX: { + auto tempconf = std::make_shared(); + tempconf->nlist = 100; + tempconf->nprobe = 16; + tempconf->nbits = 8; + tempconf->metric_type = zilliz::knowhere::METRICTYPE::L2; + return tempconf; + } + case IndexType::FAISS_IVFPQ_CPU: + case IndexType::FAISS_IVFPQ_GPU: { + auto tempconf = std::make_shared(); + tempconf->nlist = 100; + tempconf->nprobe = 16; + tempconf->nbits = 8; + tempconf->m = 8; + tempconf->metric_type = zilliz::knowhere::METRICTYPE::L2; + return tempconf; + } + case IndexType::NSG_MIX: { + auto tempconf = std::make_shared(); + tempconf->nlist = 100; + tempconf->nprobe = 16; + tempconf->search_length = 8; + tempconf->knng = 200; + tempconf->search_length = 40; // TODO(linxj): be 20 when search + tempconf->out_degree = 60; + tempconf->candidate_pool_size = 200; + tempconf->metric_type = zilliz::knowhere::METRICTYPE::L2; + return tempconf; + } + } + } +}; + class KnowhereWrapperTest - : public TestWithParam<::std::tuple> { + : public TestWithParam<::std::tuple> { protected: void SetUp() override { - zilliz::knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(DEVICE_ID, 1024*1024*200, 1024*1024*300, 2); + zilliz::knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(DEVICE_ID,1024*1024*200, 1024*1024*300, 2); std::string generator_type; - std::tie(index_type, generator_type, dim, nb, nq, k, train_cfg, search_cfg) = GetParam(); + std::tie(index_type, generator_type, dim, nb, nq, k) = GetParam(); auto generator = std::make_shared(); generator->GenData(dim, nb, nq, xb, xq, ids, k, gt_ids, gt_dis); index_ = GetVecIndexFactory(index_type); + + conf = ParamGenerator::GetInstance().Gen(index_type); + conf->k = k; + conf->d = dim; + conf->gpu_id = DEVICE_ID; } void TearDown() override { zilliz::knowhere::FaissGpuResourceMgr::GetInstance().Free(); @@ -81,8 +147,7 @@ class KnowhereWrapperTest protected: IndexType index_type; - Config train_cfg; - Config search_cfg; + Config conf; int dim = DIM; int nb = NB; @@ -102,53 +167,15 @@ class KnowhereWrapperTest INSTANTIATE_TEST_CASE_P(WrapperParam, KnowhereWrapperTest, Values( //["Index type", "Generator type", "dim", "nb", "nq", "k", "build config", "search config"] - std::make_tuple(IndexType::FAISS_IVFFLAT_CPU, "Default", - 64, 100000, 10, 10, - Config::object{{"nlist", 100}, {"dim", 64}, {"metric_type", "L2"}}, - Config::object{{"dim", 64}, {"k", 10}, {"nprobe", 10}} - ), - // to_gpu_test Failed - std::make_tuple(IndexType::FAISS_IVFFLAT_GPU, "Default", - DIM, NB, 10, 10, - Config::object{{"nlist", 100}, {"dim", DIM}, {"metric_type", "L2"}, {"gpu_id", DEVICE_ID}}, - Config::object{{"dim", DIM}, {"k", 10}, {"nprobe", 40}} - ), - std::make_tuple(IndexType::FAISS_IVFFLAT_MIX, "Default", - 64, 100000, 10, 10, - Config::object{{"nlist", 1000}, {"dim", 64}, {"metric_type", "L2"}}, - Config::object{{"dim", 64}, {"k", 10}, {"nprobe", 5}} - ), - std::make_tuple(IndexType::FAISS_IDMAP, "Default", - 64, 100000, 10, 10, - Config::object{{"dim", 64}, {"metric_type", "L2"}}, - Config::object{{"dim", 64}, {"k", 10}} - ), - std::make_tuple(IndexType::FAISS_IVFSQ8_CPU, "Default", - DIM, NB, 10, 10, - Config::object{{"dim", DIM}, {"nlist", 1000}, {"nbits", 8}, {"metric_type", "L2"}, {"gpu_id", DEVICE_ID}}, - Config::object{{"dim", DIM}, {"k", 10}, {"nprobe", 5}} - ), - std::make_tuple(IndexType::FAISS_IVFSQ8_GPU, "Default", - DIM, NB, 10, 10, - Config::object{{"dim", DIM}, {"nlist", 1000}, {"nbits", 8}, {"metric_type", "L2"}, {"gpu_id", DEVICE_ID}}, - Config::object{{"dim", DIM}, {"k", 10}, {"nprobe", 5}} - ), - std::make_tuple(IndexType::FAISS_IVFSQ8_MIX, "Default", - DIM, NB, 10, 10, - Config::object{{"dim", DIM}, {"nlist", 1000}, {"nbits", 8}, {"metric_type", "L2"}, {"gpu_id", DEVICE_ID}}, - Config::object{{"dim", DIM}, {"k", 10}, {"nprobe", 5}} - ) -// std::make_tuple(IndexType::NSG_MIX, "Default", -// 128, 250000, 10, 10, -// Config::object{{"dim", 128}, {"nlist", 8192}, {"nprobe", 16}, {"metric_type", "L2"}, -// {"knng", 200}, {"search_length", 40}, {"out_degree", 60}, {"candidate_pool_size", 200}}, -// Config::object{{"k", 10}, {"search_length", 20}} -// ) - //std::make_tuple(IndexType::SPTAG_KDT_RNT_CPU, "Default", - // 64, 10000, 10, 10, - // Config::object{{"TPTNumber", 1}, {"dim", 64}}, - // Config::object{{"dim", 64}, {"k", 10}} - //) + std::make_tuple(IndexType::FAISS_IVFFLAT_CPU, "Default", 64, 100000, 10, 10), + std::make_tuple(IndexType::FAISS_IVFFLAT_GPU, "Default", DIM, NB, 10, 10), + std::make_tuple(IndexType::FAISS_IVFFLAT_MIX, "Default", 64, 100000, 10, 10), + std::make_tuple(IndexType::FAISS_IVFSQ8_CPU, "Default", DIM, NB, 10, 10), + std::make_tuple(IndexType::FAISS_IVFSQ8_GPU, "Default", DIM, NB, 10, 10), + std::make_tuple(IndexType::FAISS_IVFSQ8_MIX, "Default", DIM, NB, 10, 10), +// std::make_tuple(IndexType::NSG_MIX, "Default", 128, 250000, 10, 10), +// std::make_tuple(IndexType::SPTAG_KDT_RNT_CPU, "Default", 128, 250000, 10, 10), + std::make_tuple(IndexType::FAISS_IDMAP, "Default", 64, 100000, 10, 10) ) ); @@ -159,8 +186,8 @@ TEST_P(KnowhereWrapperTest, BASE_TEST) { std::vector res_ids(elems); std::vector res_dis(elems); - index_->BuildAll(nb, xb.data(), ids.data(), train_cfg); - index_->Search(nq, xq.data(), res_dis.data(), res_ids.data(), search_cfg); + index_->BuildAll(nb, xb.data(), ids.data(), conf); + index_->Search(nq, xq.data(), res_dis.data(), res_ids.data(), conf); AssertResult(res_ids, res_dis); } @@ -171,14 +198,14 @@ TEST_P(KnowhereWrapperTest, TO_GPU_TEST) { std::vector res_ids(elems); std::vector res_dis(elems); - index_->BuildAll(nb, xb.data(), ids.data(), train_cfg); - index_->Search(nq, xq.data(), res_dis.data(), res_ids.data(), search_cfg); + index_->BuildAll(nb, xb.data(), ids.data(), conf); + index_->Search(nq, xq.data(), res_dis.data(), res_ids.data(), conf); AssertResult(res_ids, res_dis); { auto dev_idx = index_->CopyToGpu(DEVICE_ID); for (int i = 0; i < 10; ++i) { - dev_idx->Search(nq, xq.data(), res_dis.data(), res_ids.data(), search_cfg); + dev_idx->Search(nq, xq.data(), res_dis.data(), res_ids.data(), conf); } AssertResult(res_ids, res_dis); } @@ -190,15 +217,15 @@ TEST_P(KnowhereWrapperTest, TO_GPU_TEST) { auto dev_idx = new_index->CopyToGpu(DEVICE_ID); for (int i = 0; i < 10; ++i) { - dev_idx->Search(nq, xq.data(), res_dis.data(), res_ids.data(), search_cfg); + dev_idx->Search(nq, xq.data(), res_dis.data(), res_ids.data(), conf); } AssertResult(res_ids, res_dis); } } -TEST_P(KnowhereWrapperTest, TO_CPU_TEST) { - // dev -} +//TEST_P(KnowhereWrapperTest, TO_CPU_TEST) { +// // dev +//} TEST_P(KnowhereWrapperTest, SERIALIZE_TEST) { EXPECT_EQ(index_->GetType(), index_type); @@ -206,8 +233,8 @@ TEST_P(KnowhereWrapperTest, SERIALIZE_TEST) { auto elems = nq * k; std::vector res_ids(elems); std::vector res_dis(elems); - index_->BuildAll(nb, xb.data(), ids.data(), train_cfg); - index_->Search(nq, xq.data(), res_dis.data(), res_ids.data(), search_cfg); + index_->BuildAll(nb, xb.data(), ids.data(), conf); + index_->Search(nq, xq.data(), res_dis.data(), res_ids.data(), conf); AssertResult(res_ids, res_dis); { @@ -220,7 +247,7 @@ TEST_P(KnowhereWrapperTest, SERIALIZE_TEST) { std::vector res_ids(elems); std::vector res_dis(elems); - new_index->Search(nq, xq.data(), res_dis.data(), res_ids.data(), search_cfg); + new_index->Search(nq, xq.data(), res_dis.data(), res_ids.data(), conf); AssertResult(res_ids, res_dis); } @@ -234,7 +261,7 @@ TEST_P(KnowhereWrapperTest, SERIALIZE_TEST) { std::vector res_ids(elems); std::vector res_dis(elems); - new_index->Search(nq, xq.data(), res_dis.data(), res_ids.data(), search_cfg); + new_index->Search(nq, xq.data(), res_dis.data(), res_ids.data(), conf); AssertResult(res_ids, res_dis); } } -- GitLab