MS-565 remove jsoncons v2

Former-commit-id: 64e514a130640eae43c4e9657a3fcbe2efac3074

cpp/src/core/knowhere/CMakeLists.txt

@@ -117,7 +117,6 @@ set(CORE_INCLUDE_DIRS
         ${CORE_SOURCE_DIR}/knowhere
         ${CORE_SOURCE_DIR}/thirdparty
         ${CORE_SOURCE_DIR}/thirdparty/SPTAG/AnnService
-        ${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include
         ${ARROW_INCLUDE_DIR}
         ${FAISS_INCLUDE_DIR}
         ${OPENBLAS_INCLUDE_DIR}
@@ -129,8 +128,6 @@ set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE)
 
 #INSTALL(DIRECTORY
 #        ${CORE_SOURCE_DIR}/include/knowhere
-#        ${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons
-#        ${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons_ext
 #        ${ARROW_INCLUDE_DIR}/arrow
 #        ${FAISS_PREFIX}/include/faiss
 #        ${OPENBLAS_INCLUDE_DIR}/

cpp/src/core/test/test_ivf.cpp

@@ -146,8 +146,8 @@ INSTANTIATE_TEST_CASE_P(IVFParameters, IVFTest,
                         Values(
                             std::make_tuple("IVF", ParameterType::ivf),
                             std::make_tuple("GPUIVF", ParameterType::ivf),
-                            std::make_tuple("IVFPQ", ParameterType::ivfpq),
-                            std::make_tuple("GPUIVFPQ", ParameterType::ivfpq),
+//                            std::make_tuple("IVFPQ", ParameterType::ivfpq),
+//                            std::make_tuple("GPUIVFPQ", ParameterType::ivfpq),
                             std::make_tuple("IVFSQ", ParameterType::ivfsq),
                             std::make_tuple("GPUIVFSQ", ParameterType::ivfsq)
                         )

cpp/src/db/engine/ExecutionEngineImpl.cpp

@@ -26,6 +26,9 @@
 #include "src/wrapper/VecIndex.h"
 #include "src/wrapper/VecImpl.h"
 #include "knowhere/common/Exception.h"
+#include "knowhere/common/Config.h"
+#include "wrapper/ConfAdapterMgr.h"
+#include "wrapper/ConfAdapter.h"
 #include "server/Config.h"
 
 #include <stdexcept>
@@ -50,11 +53,15 @@ ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension,
         throw Exception(DB_ERROR, "Could not create VecIndex");
     }
 
-    Config build_cfg;
-    build_cfg["dim"] = dimension;
-    build_cfg["metric_type"] = (metric_type_ == MetricType::IP) ? "IP" : "L2";
-    AutoGenParams(index_->GetType(), 0, build_cfg);
-    auto ec = std::static_pointer_cast<BFIndex>(index_)->Build(build_cfg);
+    TempMetaConf temp_conf;
+    temp_conf.gpu_id = gpu_num_;
+    temp_conf.dim = dimension;
+    temp_conf.metric_type = (metric_type_ == MetricType::IP) ?
+                            knowhere::METRICTYPE::IP : knowhere::METRICTYPE::L2;
+    auto adapter = AdapterMgr::GetInstance().GetAdapter(index_->GetType());
+    auto conf = adapter->Match(temp_conf);
+
+    auto ec = std::static_pointer_cast<BFIndex>(index_)->Build(conf);
     if (ec != KNOWHERE_SUCCESS) {
         throw Exception(DB_ERROR, "Build index error");
     }
@@ -274,17 +281,21 @@ ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_t
         throw Exception(DB_ERROR, "Could not create VecIndex");
     }
 
-    Config build_cfg;
-    build_cfg["dim"] = Dimension();
-    build_cfg["metric_type"] = (metric_type_ == MetricType::IP) ? "IP" : "L2";
-    build_cfg["gpu_id"] = gpu_num_;
-    build_cfg["nlist"] = nlist_;
-    AutoGenParams(to_index->GetType(), Count(), build_cfg);
+    TempMetaConf temp_conf;
+    temp_conf.gpu_id = gpu_num_;
+    temp_conf.dim = Dimension();
+    temp_conf.nlist = nlist_;
+    temp_conf.metric_type = (metric_type_ == MetricType::IP) ?
+        knowhere::METRICTYPE::IP : knowhere::METRICTYPE::L2;
+    temp_conf.size = Count();
+
+    auto adapter = AdapterMgr::GetInstance().GetAdapter(to_index->GetType());
+    auto conf = adapter->Match(temp_conf);
 
     auto status = to_index->BuildAll(Count(),
-                                 from_index->GetRawVectors(),
-                                 from_index->GetRawIds(),
-                                 build_cfg);
+                                     from_index->GetRawVectors(),
+                                     from_index->GetRawIds(),
+                                     conf);
     if (!status.ok()) { throw Exception(DB_ERROR, status.message()); }
 
     return std::make_shared<ExecutionEngineImpl>(to_index, location, engine_type, metric_type_, nlist_);
@@ -302,8 +313,16 @@ Status ExecutionEngineImpl::Search(long n,
     }
 
     ENGINE_LOG_DEBUG << "Search Params: [k]  " << k << " [nprobe] " << nprobe;
-    auto cfg = Config::object{{"k", k}, {"nprobe", nprobe}};
-    auto status = index_->Search(n, data, distances, labels, cfg);
+
+    // TODO(linxj): remove here. Get conf from function
+    TempMetaConf temp_conf;
+    temp_conf.k = k;
+    temp_conf.nprobe = nprobe;
+
+    auto adapter = AdapterMgr::GetInstance().GetAdapter(index_->GetType());
+    auto conf = adapter->MatchSearch(temp_conf, index_->GetType());
+
+    auto status = index_->Search(n, data, distances, labels, conf);
     if (!status.ok()) {
         ENGINE_LOG_ERROR << "Search error";
     }

cpp/src/wrapper/ConfAdapter.cpp

+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#include <cmath>
+#include "ConfAdapter.h"
+#include "src/utils/Log.h"
+#include "knowhere/index/vector_index/helpers/IndexParameter.h"
+
+// TODO: add conf checker
+
+namespace zilliz {
+namespace milvus {
+namespace engine {
+
+#if CUDA_VERSION > 9000
+#define GPU_MAX_NRPOBE 2048
+#else
+#define GPU_MAX_NRPOBE 1024
+#endif
+
+void
+ConfAdapter::MatchBase(knowhere::Config conf) {
+    if (conf->metric_type == knowhere::DEFAULT_TYPE)
+        conf->metric_type = knowhere::METRICTYPE::L2;
+    if (conf->gpu_id == knowhere::INVALID_VALUE)
+        conf->gpu_id = 0;
+}
+
+knowhere::Config
+ConfAdapter::Match(const TempMetaConf &metaconf) {
+    auto conf = std::make_shared<knowhere::Cfg>();
+    conf->d = metaconf.dim;
+    conf->metric_type = metaconf.metric_type;
+    conf->gpu_id = conf->gpu_id;
+    MatchBase(conf);
+    return conf;
+}
+
+knowhere::Config
+ConfAdapter::MatchSearch(const TempMetaConf &metaconf, const IndexType &type) {
+    auto conf = std::make_shared<knowhere::Cfg>();
+    conf->k = metaconf.k;
+    return conf;
+}
+
+knowhere::Config
+IVFConfAdapter::Match(const TempMetaConf &metaconf) {
+    auto conf = std::make_shared<knowhere::IVFCfg>();
+    conf->nlist = MatchNlist(metaconf.size, metaconf.nlist);
+    conf->d = metaconf.dim;
+    conf->metric_type = metaconf.metric_type;
+    conf->gpu_id = conf->gpu_id;
+    MatchBase(conf);
+    return conf;
+}
+
+static constexpr float TYPICAL_COUNT = 1000000.0;
+
+int64_t
+IVFConfAdapter::MatchNlist(const int64_t &size, const int64_t &nlist) {
+    if (size <= TYPICAL_COUNT / 16384 + 1) {
+        // handle less row count, avoid nlist set to 0
+        return 1;
+    } else if (int(size / TYPICAL_COUNT) * nlist == 0) {
+        // calculate a proper nlist if nlist not specified or size less than TYPICAL_COUNT
+        return int(size / TYPICAL_COUNT * 16384);
+    }
+    return 0;
+}
+
+knowhere::Config
+IVFConfAdapter::MatchSearch(const TempMetaConf &metaconf, const IndexType &type) {
+    auto conf = std::make_shared<knowhere::IVFCfg>();
+    conf->k = metaconf.k;
+    conf->nprobe = metaconf.nprobe;
+
+    switch (type) {
+        case IndexType::FAISS_IVFFLAT_GPU:
+        case IndexType::FAISS_IVFSQ8_GPU:
+        case IndexType::FAISS_IVFPQ_GPU:
+            if (conf->nprobe > GPU_MAX_NRPOBE) {
+                WRAPPER_LOG_WARNING << "When search with GPU, nprobe shoud be no more than " << GPU_MAX_NRPOBE
+                                    << ", but you passed " << conf->nprobe
+                                    << ". Search with " << GPU_MAX_NRPOBE << " instead";
+                conf->nprobe = GPU_MAX_NRPOBE;
+
+            }
+    }
+    return conf;
+}
+
+knowhere::Config
+IVFSQConfAdapter::Match(const TempMetaConf &metaconf) {
+    auto conf = std::make_shared<knowhere::IVFSQCfg>();
+    conf->nlist = MatchNlist(metaconf.size, metaconf.nlist);
+    conf->d = metaconf.dim;
+    conf->metric_type = metaconf.metric_type;
+    conf->gpu_id = conf->gpu_id;
+    conf->nbits = 8;
+    MatchBase(conf);
+    return conf;
+}
+
+knowhere::Config
+IVFPQConfAdapter::Match(const TempMetaConf &metaconf) {
+    auto conf = std::make_shared<knowhere::IVFPQCfg>();
+    conf->nlist = MatchNlist(metaconf.size, metaconf.nlist);
+    conf->d = metaconf.dim;
+    conf->metric_type = metaconf.metric_type;
+    conf->gpu_id = conf->gpu_id;
+    conf->nbits = 8;
+    conf->m = 8;
+    MatchBase(conf);
+    return conf;
+}
+
+knowhere::Config
+NSGConfAdapter::Match(const TempMetaConf &metaconf) {
+    auto conf = std::make_shared<knowhere::NSGCfg>();
+    conf->nlist = MatchNlist(metaconf.size, metaconf.nlist);
+    conf->d = metaconf.dim;
+    conf->metric_type = metaconf.metric_type;
+    conf->gpu_id = conf->gpu_id;
+
+    auto scale_factor = round(metaconf.dim / 128.0);
+    scale_factor = scale_factor >= 4 ? 4 : scale_factor;
+    conf->nprobe = 6 + 10 * scale_factor;
+    conf->knng = 100 + 100 * scale_factor;
+    conf->search_length = 40 + 5 * scale_factor;
+    conf->out_degree = 50 + 5 * scale_factor;
+    conf->candidate_pool_size = 200 + 100 * scale_factor;
+    MatchBase(conf);
+    return conf;
+}
+
+knowhere::Config
+NSGConfAdapter::MatchSearch(const TempMetaConf &metaconf, const IndexType &type) {
+    auto conf = std::make_shared<knowhere::NSGCfg>();
+    conf->k = metaconf.k;
+    conf->search_length = metaconf.search_length;
+    return conf;
+}
+
+}
+}
+}

cpp/src/wrapper/ConfAdapter.h

+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#pragma once
+
+#include "knowhere/common/Config.h"
+#include "VecIndex.h"
+
+
+namespace zilliz {
+namespace milvus {
+namespace engine {
+
+// TODO(linxj): remove later, replace with real metaconf
+constexpr int64_t TEMPMETA_DEFAULT_VALUE = -1;
+struct TempMetaConf {
+    int64_t size = TEMPMETA_DEFAULT_VALUE;
+    int64_t nlist = TEMPMETA_DEFAULT_VALUE;
+    int64_t dim = TEMPMETA_DEFAULT_VALUE;
+    int64_t gpu_id = TEMPMETA_DEFAULT_VALUE;
+    int64_t k = TEMPMETA_DEFAULT_VALUE;
+    int64_t nprobe = TEMPMETA_DEFAULT_VALUE;
+    int64_t search_length = TEMPMETA_DEFAULT_VALUE;
+    knowhere::METRICTYPE metric_type = knowhere::DEFAULT_TYPE;
+};
+
+class ConfAdapter {
+ public:
+    virtual knowhere::Config
+    Match(const TempMetaConf &metaconf);
+
+    virtual knowhere::Config
+    MatchSearch(const TempMetaConf &metaconf, const IndexType &type);
+
+ protected:
+    static void MatchBase(knowhere::Config conf);
+};
+
+using ConfAdapterPtr = std::shared_ptr<ConfAdapter>;
+
+class IVFConfAdapter : public ConfAdapter {
+ public:
+    knowhere::Config
+    Match(const TempMetaConf &metaconf) override;
+
+    knowhere::Config
+    MatchSearch(const TempMetaConf &metaconf, const IndexType &type) override;
+
+ protected:
+    static int64_t MatchNlist(const int64_t &size, const int64_t &nlist);
+};
+
+class IVFSQConfAdapter : public IVFConfAdapter {
+ public:
+    knowhere::Config
+    Match(const TempMetaConf &metaconf) override;
+};
+
+class IVFPQConfAdapter : public IVFConfAdapter {
+ public:
+    knowhere::Config
+    Match(const TempMetaConf &metaconf) override;
+};
+
+class NSGConfAdapter : public IVFConfAdapter {
+ public:
+    knowhere::Config
+    Match(const TempMetaConf &metaconf) override;
+
+    knowhere::Config
+    MatchSearch(const TempMetaConf &metaconf, const IndexType &type) final;
+};
+
+}
+}
+}

cpp/src/wrapper/ConfAdapterMgr.cpp

+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#include "src/utils/Exception.h"
+#include "ConfAdapterMgr.h"
+
+
+namespace zilliz {
+namespace milvus {
+namespace engine {
+
+ConfAdapterPtr
+AdapterMgr::GetAdapter(const IndexType &indexType) {
+    if (!init_) RegisterAdapter();
+
+    auto it = table_.find(indexType);
+    if (it != table_.end()) {
+        return it->second();
+    } else {
+        throw Exception(KNOWHERE_INVALID_ARGUMENT, "Can not find this type of confadapter");
+    }
+}
+
+
+#define REGISTER_CONF_ADAPTER(T, KEY, NAME) static AdapterMgr::register_t<T>reg_##NAME##_(KEY)
+void
+AdapterMgr::RegisterAdapter() {
+    init_ = true;
+
+    REGISTER_CONF_ADAPTER(ConfAdapter, IndexType::FAISS_IDMAP, idmap);
+
+    REGISTER_CONF_ADAPTER(IVFConfAdapter, IndexType::FAISS_IVFFLAT_CPU, ivf_cpu);
+    REGISTER_CONF_ADAPTER(IVFConfAdapter, IndexType::FAISS_IVFFLAT_GPU, ivf_gpu);
+    REGISTER_CONF_ADAPTER(IVFConfAdapter, IndexType::FAISS_IVFFLAT_MIX, ivf_mix);
+
+    REGISTER_CONF_ADAPTER(IVFSQConfAdapter, IndexType::FAISS_IVFSQ8_CPU, ivfsq8_cpu);
+    REGISTER_CONF_ADAPTER(IVFSQConfAdapter, IndexType::FAISS_IVFSQ8_GPU, ivfsq8_gpu);
+    REGISTER_CONF_ADAPTER(IVFSQConfAdapter, IndexType::FAISS_IVFSQ8_MIX, ivfsq8_mix);
+
+    REGISTER_CONF_ADAPTER(IVFPQConfAdapter, IndexType::FAISS_IVFPQ_CPU, ivfpq_cpu);
+    REGISTER_CONF_ADAPTER(IVFPQConfAdapter, IndexType::FAISS_IVFPQ_GPU, ivfpq_gpu);
+
+    REGISTER_CONF_ADAPTER(NSGConfAdapter, IndexType::NSG_MIX, nsg_mix);
+}
+
+} // engine
+} // milvus
+} // zilliz
+

cpp/src/wrapper/ConfAdapterMgr.h

+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#pragma once
+
+#include "VecIndex.h"
+#include "ConfAdapter.h"
+
+
+namespace zilliz {
+namespace milvus {
+namespace engine {
+
+class AdapterMgr {
+ public:
+    template<typename T>
+    struct register_t {
+        explicit register_t(const IndexType &key) {
+            AdapterMgr::GetInstance().table_.emplace(key, [] {
+                return std::make_shared<T>();
+            });
+        }
+    };
+
+    static AdapterMgr &
+    GetInstance() {
+        static AdapterMgr instance;
+        return instance;
+    }
+
+    ConfAdapterPtr
+    GetAdapter(const IndexType &indexType);
+
+    void
+    RegisterAdapter();
+
+ protected:
+    bool init_ = false;
+    std::map<IndexType, std::function<ConfAdapterPtr()> > table_;
+};
+
+
+} // engine
+} // milvus
+} // zilliz
+
+
+

cpp/src/wrapper/VecImpl.cpp

@@ -24,6 +24,10 @@
 #include "VecImpl.h"
 #include "DataTransfer.h"
 
+/*
+ * no parameter check in this layer.
+ * only responible for index combination
+ */
 
 namespace zilliz {
 namespace milvus {
@@ -39,7 +43,7 @@ VecIndexImpl::BuildAll(const long &nb,
                        const long &nt,
                        const float *xt) {
     try {
-        dim = cfg["dim"].as<int>();
+        dim = cfg->d;
         auto dataset = GenDatasetWithIds(nb, dim, xb, ids);
 
         auto preprocessor = index_->BuildPreprocessor(dataset, cfg);
@@ -50,9 +54,6 @@ VecIndexImpl::BuildAll(const long &nb,
     } catch (KnowhereException &e) {
         WRAPPER_LOG_ERROR << e.what();
         return Status(KNOWHERE_UNEXPECTED_ERROR, e.what());
-    } catch (jsoncons::json_exception &e) {
-        WRAPPER_LOG_ERROR << e.what();
-        return Status(KNOWHERE_INVALID_ARGUMENT, e.what());
     } catch (std::exception &e) {
         WRAPPER_LOG_ERROR << e.what();
         return Status(KNOWHERE_ERROR, e.what());
@@ -69,9 +70,6 @@ VecIndexImpl::Add(const long &nb, const float *xb, const long *ids, const Config
     } catch (KnowhereException &e) {
         WRAPPER_LOG_ERROR << e.what();
         return Status(KNOWHERE_UNEXPECTED_ERROR, e.what());
-    } catch (jsoncons::json_exception &e) {
-        WRAPPER_LOG_ERROR << e.what();
-        return Status(KNOWHERE_INVALID_ARGUMENT, e.what());
     } catch (std::exception &e) {
         WRAPPER_LOG_ERROR << e.what();
         return Status(KNOWHERE_ERROR, e.what());
@@ -82,13 +80,11 @@ VecIndexImpl::Add(const long &nb, const float *xb, const long *ids, const Config
 Status
 VecIndexImpl::Search(const long &nq, const float *xq, float *dist, long *ids, const Config &cfg) {
     try {
-        auto k = cfg["k"].as<int>();
+        auto k = cfg->k;
         auto dataset = GenDataset(nq, dim, xq);
 
         Config search_cfg = cfg;
 
-        ParameterValidation(type, search_cfg);
-
         auto res = index_->Search(dataset, search_cfg);
         auto ids_array = res->array()[0];
         auto dis_array = res->array()[1];
@@ -120,9 +116,6 @@ VecIndexImpl::Search(const long &nq, const float *xq, float *dist, long *ids, co
     } catch (KnowhereException &e) {
         WRAPPER_LOG_ERROR << e.what();
         return Status(KNOWHERE_UNEXPECTED_ERROR, e.what());
-    } catch (jsoncons::json_exception &e) {
-        WRAPPER_LOG_ERROR << e.what();
-        return Status(KNOWHERE_INVALID_ARGUMENT, e.what());
     } catch (std::exception &e) {
         WRAPPER_LOG_ERROR << e.what();
         return Status(KNOWHERE_ERROR, e.what());
@@ -208,14 +201,11 @@ BFIndex::GetRawIds() {
 ErrorCode
 BFIndex::Build(const Config &cfg) {
     try {
-        dim = cfg["dim"].as<int>();
+        dim = cfg->d;
         std::static_pointer_cast<IDMAP>(index_)->Train(cfg);
     } catch (KnowhereException &e) {
         WRAPPER_LOG_ERROR << e.what();
         return KNOWHERE_UNEXPECTED_ERROR;
-    } catch (jsoncons::json_exception &e) {
-        WRAPPER_LOG_ERROR << e.what();
-        return KNOWHERE_INVALID_ARGUMENT;
     } catch (std::exception &e) {
         WRAPPER_LOG_ERROR << e.what();
         return KNOWHERE_ERROR;
@@ -231,7 +221,7 @@ BFIndex::BuildAll(const long &nb,
                   const long &nt,
                   const float *xt) {
     try {
-        dim = cfg["dim"].as<int>();
+        dim = cfg->d;
         auto dataset = GenDatasetWithIds(nb, dim, xb, ids);
 
         std::static_pointer_cast<IDMAP>(index_)->Train(cfg);
@@ -239,9 +229,6 @@ BFIndex::BuildAll(const long &nb,
     } catch (KnowhereException &e) {
         WRAPPER_LOG_ERROR << e.what();
         return Status(KNOWHERE_UNEXPECTED_ERROR, e.what());
-    } catch (jsoncons::json_exception &e) {
-        WRAPPER_LOG_ERROR << e.what();
-        return Status(KNOWHERE_INVALID_ARGUMENT, e.what());
     } catch (std::exception &e) {
         WRAPPER_LOG_ERROR << e.what();
         return Status(KNOWHERE_ERROR, e.what());
@@ -258,7 +245,7 @@ IVFMixIndex::BuildAll(const long &nb,
                       const long &nt,
                       const float *xt) {
     try {
-        dim = cfg["dim"].as<int>();
+        dim = cfg->d;
         auto dataset = GenDatasetWithIds(nb, dim, xb, ids);
 
         auto preprocessor = index_->BuildPreprocessor(dataset, cfg);
@@ -278,9 +265,6 @@ IVFMixIndex::BuildAll(const long &nb,
     } catch (KnowhereException &e) {
         WRAPPER_LOG_ERROR << e.what();
         return Status(KNOWHERE_UNEXPECTED_ERROR, e.what());
-    } catch (jsoncons::json_exception &e) {
-        WRAPPER_LOG_ERROR << e.what();
-        return Status(KNOWHERE_INVALID_ARGUMENT, e.what());
     } catch (std::exception &e) {
         WRAPPER_LOG_ERROR << e.what();
         return Status(KNOWHERE_ERROR, e.what());
@@ -290,7 +274,6 @@ IVFMixIndex::BuildAll(const long &nb,
 
 Status
 IVFMixIndex::Load(const zilliz::knowhere::BinarySet &index_binary) {
-    //index_ = std::make_shared<IVF>();
     index_->Load(index_binary);
     dim = Dimension();
     return Status::OK();

cpp/src/wrapper/VecIndex.cpp

@@ -36,8 +36,6 @@ namespace zilliz {
 namespace milvus {
 namespace engine {
 
-static constexpr float TYPICAL_COUNT = 1000000.0;
-
 struct FileIOReader {
     std::fstream fs;
     std::string name;
@@ -101,7 +99,7 @@ FileIOWriter::operator()(void *ptr, size_t size) {
 VecIndexPtr
 GetVecIndexFactory(const IndexType &type, const Config &cfg) {
     std::shared_ptr<zilliz::knowhere::VectorIndex> index;
-    auto gpu_device = cfg.get_with_default("gpu_id", 0);
+    auto gpu_device = -1; // TODO(linxj): remove hardcode here
     switch (type) {
         case IndexType::FAISS_IDMAP: {
             index = std::make_shared<zilliz::knowhere::IDMAP>();
@@ -112,12 +110,11 @@ GetVecIndexFactory(const IndexType &type, const Config &cfg) {
             break;
         }
         case IndexType::FAISS_IVFFLAT_GPU: {
-            // TODO(linxj): 规范化参数
             index = std::make_shared<zilliz::knowhere::GPUIVF>(gpu_device);
             break;
         }
         case IndexType::FAISS_IVFFLAT_MIX: {
-            index = std::make_shared<zilliz::knowhere::GPUIVF>(0);
+            index = std::make_shared<zilliz::knowhere::GPUIVF>(gpu_device);
             return std::make_shared<IVFMixIndex>(index, IndexType::FAISS_IVFFLAT_MIX);
         }
         case IndexType::FAISS_IVFPQ_CPU: {
@@ -242,70 +239,6 @@ write_index(VecIndexPtr index, const std::string &location) {
     return Status::OK();
 }
 
-
-// TODO(linxj): redo here.
-void
-AutoGenParams(const IndexType &type, const long &size, zilliz::knowhere::Config &cfg) {
-    auto nlist = cfg.get_with_default("nlist", 0);
-    if (size <= TYPICAL_COUNT / 16384 + 1) {
-        //handle less row count, avoid nlist set to 0
-        cfg["nlist"] = 1;
-    } else if (int(size / TYPICAL_COUNT) * nlist == 0) {
-        //calculate a proper nlist if nlist not specified or size less than TYPICAL_COUNT
-        cfg["nlist"] = int(size / TYPICAL_COUNT * 16384);
-    }
-
-    if (!cfg.contains("gpu_id")) { cfg["gpu_id"] = int(0); }
-    if (!cfg.contains("metric_type")) { cfg["metric_type"] = "L2"; }
-
-    switch (type) {
-        case IndexType::FAISS_IVFSQ8_MIX: {
-            if (!cfg.contains("nbits")) { cfg["nbits"] = int(8); }
-            break;
-        }
-        case IndexType::NSG_MIX: {
-            auto scale_factor = round(cfg["dim"].as<int>() / 128.0);
-            scale_factor = scale_factor >= 4 ? 4 : scale_factor;
-            cfg["nlist"] = int(size / 1000000.0 * 8192);
-            if (!cfg.contains("nprobe")) { cfg["nprobe"] = 6 + 10 * scale_factor; }
-            if (!cfg.contains("knng")) { cfg["knng"] = 100 + 100 * scale_factor; }
-            if (!cfg.contains("search_length")) { cfg["search_length"] = 40 + 5 * scale_factor; }
-            if (!cfg.contains("out_degree")) { cfg["out_degree"] = 50 + 5 * scale_factor; }
-            if (!cfg.contains("candidate_pool_size")) { cfg["candidate_pool_size"] = 200 + 100 * scale_factor; }
-            WRAPPER_LOG_DEBUG << pretty_print(cfg);
-            break;
-        }
-    }
-}
-
-#if CUDA_VERSION > 9000
-#define GPU_MAX_NRPOBE 2048
-#else
-#define GPU_MAX_NRPOBE 1024
-#endif
-
-void
-ParameterValidation(const IndexType &type, Config &cfg) {
-    switch (type) {
-        case IndexType::FAISS_IVFSQ8_GPU:
-        case IndexType::FAISS_IVFFLAT_GPU:
-        case IndexType::FAISS_IVFPQ_GPU: {
-            //search on GPU
-            if (cfg.get_with_default("nprobe", 0) != 0) {
-                auto nprobe = cfg["nprobe"].as<int>();
-                if (nprobe > GPU_MAX_NRPOBE) {
-                    WRAPPER_LOG_WARNING << "When search with GPU, nprobe shoud be no more than " << GPU_MAX_NRPOBE
-                                        << ", but you passed " << nprobe
-                                        << ". Search with " << GPU_MAX_NRPOBE << " instead";
-                    cfg.insert_or_assign("nprobe", GPU_MAX_NRPOBE);
-                }
-            }
-            break;
-        }
-        default:break;
-    }
-}
-
 IndexType
 ConvertToCpuIndexType(const IndexType &type) {
     // TODO(linxj): add IDMAP

cpp/src/wrapper/VecIndex.h

@@ -30,7 +30,6 @@ namespace zilliz {
 namespace milvus {
 namespace engine {
 
-// TODO(linxj): jsoncons => rapidjson or other.
 using Config = zilliz::knowhere::Config;
 
 enum class IndexType {
@@ -116,12 +115,6 @@ GetVecIndexFactory(const IndexType &type, const Config &cfg = Config());
 extern VecIndexPtr
 LoadVecIndex(const IndexType &index_type, const zilliz::knowhere::BinarySet &index_binary);
 
-extern void
-AutoGenParams(const IndexType &type, const long &size, Config &cfg);
-
-extern void
-ParameterValidation(const IndexType &type, Config &cfg);
-
 extern IndexType
 ConvertToCpuIndexType(const IndexType &type);
 

cpp/unittest/wrapper/wrapper_test.cpp

@@ -18,10 +18,12 @@
 #include "utils/easylogging++.h"
 #include "src/wrapper/VecIndex.h"
 #include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h"
+#include "knowhere/index/vector_index/helpers/IndexParameter.h"
 #include "utils.h"
 
 #include <gtest/gtest.h>
 
+
 INITIALIZE_EASYLOGGINGPP
 
 using namespace zilliz::milvus::engine;
@@ -35,19 +37,83 @@ constexpr int64_t DIM = 128;
 constexpr int64_t NB = 100000;
 constexpr int64_t DEVICE_ID = 0;
 
+class ParamGenerator {
+ public:
+    static ParamGenerator& GetInstance(){
+        static ParamGenerator instance;
+        return instance;
+    }
+
+    Config Gen(const IndexType& type) {
+        switch (type) {
+            case IndexType::FAISS_IDMAP: {
+                auto tempconf = std::make_shared<zilliz::knowhere::Cfg>();
+                tempconf->metric_type = zilliz::knowhere::METRICTYPE::L2;
+                return tempconf;
+            }
+            case IndexType::FAISS_IVFFLAT_CPU:
+            case IndexType::FAISS_IVFFLAT_GPU:
+            case IndexType::FAISS_IVFFLAT_MIX: {
+                auto tempconf = std::make_shared<zilliz::knowhere::IVFCfg>();
+                tempconf->nlist = 100;
+                tempconf->nprobe = 16;
+                tempconf->metric_type = zilliz::knowhere::METRICTYPE::L2;
+                return tempconf;
+            }
+            case IndexType::FAISS_IVFSQ8_CPU:
+            case IndexType::FAISS_IVFSQ8_GPU:
+            case IndexType::FAISS_IVFSQ8_MIX: {
+                auto tempconf = std::make_shared<zilliz::knowhere::IVFSQCfg>();
+                tempconf->nlist = 100;
+                tempconf->nprobe = 16;
+                tempconf->nbits = 8;
+                tempconf->metric_type = zilliz::knowhere::METRICTYPE::L2;
+                return tempconf;
+            }
+            case IndexType::FAISS_IVFPQ_CPU:
+            case IndexType::FAISS_IVFPQ_GPU: {
+                auto tempconf = std::make_shared<zilliz::knowhere::IVFPQCfg>();
+                tempconf->nlist = 100;
+                tempconf->nprobe = 16;
+                tempconf->nbits = 8;
+                tempconf->m = 8;
+                tempconf->metric_type = zilliz::knowhere::METRICTYPE::L2;
+                return tempconf;
+            }
+            case IndexType::NSG_MIX: {
+                auto tempconf = std::make_shared<zilliz::knowhere::NSGCfg>();
+                tempconf->nlist = 100;
+                tempconf->nprobe = 16;
+                tempconf->search_length = 8;
+                tempconf->knng = 200;
+                tempconf->search_length = 40; // TODO(linxj): be 20 when search
+                tempconf->out_degree = 60;
+                tempconf->candidate_pool_size = 200;
+                tempconf->metric_type = zilliz::knowhere::METRICTYPE::L2;
+                return tempconf;
+            }
+        }
+    }
+};
+
 class KnowhereWrapperTest
-    : public TestWithParam<::std::tuple<IndexType, std::string, int, int, int, int, Config, Config>> {
+    : public TestWithParam<::std::tuple<IndexType, std::string, int, int, int, int>> {
  protected:
     void SetUp() override {
-        zilliz::knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(DEVICE_ID, 1024*1024*200, 1024*1024*300, 2);
+        zilliz::knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(DEVICE_ID,1024*1024*200, 1024*1024*300, 2);
 
         std::string generator_type;
-        std::tie(index_type, generator_type, dim, nb, nq, k, train_cfg, search_cfg) = GetParam();
+        std::tie(index_type, generator_type, dim, nb, nq, k) = GetParam();
 
         auto generator = std::make_shared<DataGenBase>();
         generator->GenData(dim, nb, nq, xb, xq, ids, k, gt_ids, gt_dis);
 
         index_ = GetVecIndexFactory(index_type);
+
+        conf = ParamGenerator::GetInstance().Gen(index_type);
+        conf->k = k;
+        conf->d = dim;
+        conf->gpu_id = DEVICE_ID;
     }
     void TearDown() override {
         zilliz::knowhere::FaissGpuResourceMgr::GetInstance().Free();
@@ -81,8 +147,7 @@ class KnowhereWrapperTest
 
  protected:
     IndexType index_type;
-    Config train_cfg;
-    Config search_cfg;
+    Config conf;
 
     int dim = DIM;
     int nb = NB;
@@ -102,53 +167,15 @@ class KnowhereWrapperTest
 INSTANTIATE_TEST_CASE_P(WrapperParam, KnowhereWrapperTest,
                         Values(
                             //["Index type", "Generator type", "dim", "nb", "nq", "k", "build config", "search config"]
-                            std::make_tuple(IndexType::FAISS_IVFFLAT_CPU, "Default",
-                                            64, 100000, 10, 10,
-                                            Config::object{{"nlist", 100}, {"dim", 64}, {"metric_type", "L2"}},
-                                            Config::object{{"dim", 64}, {"k", 10}, {"nprobe", 10}}
-                            ),
-                            // to_gpu_test Failed
-                            std::make_tuple(IndexType::FAISS_IVFFLAT_GPU, "Default",
-                                            DIM, NB, 10, 10,
-                                            Config::object{{"nlist", 100}, {"dim", DIM}, {"metric_type", "L2"}, {"gpu_id", DEVICE_ID}},
-                                            Config::object{{"dim", DIM}, {"k", 10}, {"nprobe", 40}}
-                            ),
-                            std::make_tuple(IndexType::FAISS_IVFFLAT_MIX, "Default",
-                                            64, 100000, 10, 10,
-                                            Config::object{{"nlist", 1000}, {"dim", 64}, {"metric_type", "L2"}},
-                                            Config::object{{"dim", 64}, {"k", 10}, {"nprobe", 5}}
-                            ),
-                            std::make_tuple(IndexType::FAISS_IDMAP, "Default",
-                                            64, 100000, 10, 10,
-                                            Config::object{{"dim", 64}, {"metric_type", "L2"}},
-                                            Config::object{{"dim", 64}, {"k", 10}}
-                            ),
-                            std::make_tuple(IndexType::FAISS_IVFSQ8_CPU, "Default",
-                                            DIM, NB, 10, 10,
-                                            Config::object{{"dim", DIM}, {"nlist", 1000}, {"nbits", 8}, {"metric_type", "L2"}, {"gpu_id", DEVICE_ID}},
-                                            Config::object{{"dim", DIM}, {"k", 10}, {"nprobe", 5}}
-                            ),
-                            std::make_tuple(IndexType::FAISS_IVFSQ8_GPU, "Default",
-                                            DIM, NB, 10, 10,
-                                            Config::object{{"dim", DIM}, {"nlist", 1000}, {"nbits", 8}, {"metric_type", "L2"}, {"gpu_id", DEVICE_ID}},
-                                            Config::object{{"dim", DIM}, {"k", 10}, {"nprobe", 5}}
-                            ),
-                            std::make_tuple(IndexType::FAISS_IVFSQ8_MIX, "Default",
-                                            DIM, NB, 10, 10,
-                                            Config::object{{"dim", DIM}, {"nlist", 1000}, {"nbits", 8}, {"metric_type", "L2"}, {"gpu_id", DEVICE_ID}},
-                                            Config::object{{"dim", DIM}, {"k", 10}, {"nprobe", 5}}
-                            )
-//                            std::make_tuple(IndexType::NSG_MIX, "Default",
-//                                            128, 250000, 10, 10,
-//                                            Config::object{{"dim", 128}, {"nlist", 8192}, {"nprobe", 16}, {"metric_type", "L2"},
-//                                                           {"knng", 200}, {"search_length", 40}, {"out_degree", 60}, {"candidate_pool_size", 200}},
-//                                            Config::object{{"k", 10}, {"search_length", 20}}
-//                            )
-                            //std::make_tuple(IndexType::SPTAG_KDT_RNT_CPU, "Default",
-                            //                64, 10000, 10, 10,
-                            //                Config::object{{"TPTNumber", 1}, {"dim", 64}},
-                            //                Config::object{{"dim", 64}, {"k", 10}}
-                            //)
+                            std::make_tuple(IndexType::FAISS_IVFFLAT_CPU, "Default", 64, 100000, 10, 10),
+                            std::make_tuple(IndexType::FAISS_IVFFLAT_GPU, "Default", DIM, NB, 10, 10),
+                            std::make_tuple(IndexType::FAISS_IVFFLAT_MIX, "Default", 64, 100000, 10, 10),
+                            std::make_tuple(IndexType::FAISS_IVFSQ8_CPU, "Default", DIM, NB, 10, 10),
+                            std::make_tuple(IndexType::FAISS_IVFSQ8_GPU, "Default", DIM, NB, 10, 10),
+                            std::make_tuple(IndexType::FAISS_IVFSQ8_MIX, "Default", DIM, NB, 10, 10),
+//                            std::make_tuple(IndexType::NSG_MIX, "Default", 128, 250000, 10, 10),
+//                            std::make_tuple(IndexType::SPTAG_KDT_RNT_CPU, "Default", 128, 250000, 10, 10),
+                            std::make_tuple(IndexType::FAISS_IDMAP, "Default", 64, 100000, 10, 10)
                         )
 );
 
@@ -159,8 +186,8 @@ TEST_P(KnowhereWrapperTest, BASE_TEST) {
     std::vector<int64_t> res_ids(elems);
     std::vector<float> res_dis(elems);
 
-    index_->BuildAll(nb, xb.data(), ids.data(), train_cfg);
-    index_->Search(nq, xq.data(), res_dis.data(), res_ids.data(), search_cfg);
+    index_->BuildAll(nb, xb.data(), ids.data(), conf);
+    index_->Search(nq, xq.data(), res_dis.data(), res_ids.data(), conf);
     AssertResult(res_ids, res_dis);
 }
 
@@ -171,14 +198,14 @@ TEST_P(KnowhereWrapperTest, TO_GPU_TEST) {
     std::vector<int64_t> res_ids(elems);
     std::vector<float> res_dis(elems);
 
-    index_->BuildAll(nb, xb.data(), ids.data(), train_cfg);
-    index_->Search(nq, xq.data(), res_dis.data(), res_ids.data(), search_cfg);
+    index_->BuildAll(nb, xb.data(), ids.data(), conf);
+    index_->Search(nq, xq.data(), res_dis.data(), res_ids.data(), conf);
     AssertResult(res_ids, res_dis);
 
     {
         auto dev_idx = index_->CopyToGpu(DEVICE_ID);
         for (int i = 0; i < 10; ++i) {
-            dev_idx->Search(nq, xq.data(), res_dis.data(), res_ids.data(), search_cfg);
+            dev_idx->Search(nq, xq.data(), res_dis.data(), res_ids.data(), conf);
         }
         AssertResult(res_ids, res_dis);
     }
@@ -190,15 +217,15 @@ TEST_P(KnowhereWrapperTest, TO_GPU_TEST) {
 
         auto dev_idx = new_index->CopyToGpu(DEVICE_ID);
         for (int i = 0; i < 10; ++i) {
-            dev_idx->Search(nq, xq.data(), res_dis.data(), res_ids.data(), search_cfg);
+            dev_idx->Search(nq, xq.data(), res_dis.data(), res_ids.data(), conf);
         }
         AssertResult(res_ids, res_dis);
     }
 }
 
-TEST_P(KnowhereWrapperTest, TO_CPU_TEST) {
-    // dev
-}
+//TEST_P(KnowhereWrapperTest, TO_CPU_TEST) {
+//    // dev
+//}
 
 TEST_P(KnowhereWrapperTest, SERIALIZE_TEST) {
     EXPECT_EQ(index_->GetType(), index_type);
@@ -206,8 +233,8 @@ TEST_P(KnowhereWrapperTest, SERIALIZE_TEST) {
     auto elems = nq * k;
     std::vector<int64_t> res_ids(elems);
     std::vector<float> res_dis(elems);
-    index_->BuildAll(nb, xb.data(), ids.data(), train_cfg);
-    index_->Search(nq, xq.data(), res_dis.data(), res_ids.data(), search_cfg);
+    index_->BuildAll(nb, xb.data(), ids.data(), conf);
+    index_->Search(nq, xq.data(), res_dis.data(), res_ids.data(), conf);
     AssertResult(res_ids, res_dis);
 
     {
@@ -220,7 +247,7 @@ TEST_P(KnowhereWrapperTest, SERIALIZE_TEST) {
 
         std::vector<int64_t> res_ids(elems);
         std::vector<float> res_dis(elems);
-        new_index->Search(nq, xq.data(), res_dis.data(), res_ids.data(), search_cfg);
+        new_index->Search(nq, xq.data(), res_dis.data(), res_ids.data(), conf);
         AssertResult(res_ids, res_dis);
     }
 
@@ -234,7 +261,7 @@ TEST_P(KnowhereWrapperTest, SERIALIZE_TEST) {
 
         std::vector<int64_t> res_ids(elems);
         std::vector<float> res_dis(elems);
-        new_index->Search(nq, xq.data(), res_dis.data(), res_ids.data(), search_cfg);
+        new_index->Search(nq, xq.data(), res_dis.data(), res_ids.data(), conf);
         AssertResult(res_ids, res_dis);
     }
 }