diff --git a/internal/core/bench/bench_indexbuilder.cpp b/internal/core/bench/bench_indexbuilder.cpp index 6b3ae5c71644c4b4c37d173854ea239b5bf3052d..570294fc439425a9c81d387779e8393f95fbb6f7 100644 --- a/internal/core/bench/bench_indexbuilder.cpp +++ b/internal/core/bench/bench_indexbuilder.cpp @@ -17,7 +17,7 @@ #include #include "pb/index_cgo_msg.pb.h" -#include "indexbuilder/IndexWrapper.h" +#include "indexbuilder/VecIndexCreator.h" #include "indexbuilder/index_c.h" #include "indexbuilder/utils.h" #include "test_utils/indexbuilder_test_utils.h" @@ -64,7 +64,7 @@ IndexBuilder_build(benchmark::State& state) { for (auto _ : state) { auto index = - std::make_unique(type_params_str.c_str(), index_params_str.c_str()); + std::make_unique(type_params_str.c_str(), index_params_str.c_str()); index->BuildWithoutIds(xb_dataset); } } @@ -93,7 +93,7 @@ IndexBuilder_build_and_codec(benchmark::State& state) { for (auto _ : state) { auto index = - std::make_unique(type_params_str.c_str(), index_params_str.c_str()); + std::make_unique(type_params_str.c_str(), index_params_str.c_str()); index->BuildWithoutIds(xb_dataset); index->Serialize(); diff --git a/internal/core/src/indexbuilder/CMakeLists.txt b/internal/core/src/indexbuilder/CMakeLists.txt index 01fcf2dc9d458d6cd58e854008618f42c22e613b..e303a0e93f51733735085e419124fbeaa8541f90 100644 --- a/internal/core/src/indexbuilder/CMakeLists.txt +++ b/internal/core/src/indexbuilder/CMakeLists.txt @@ -11,9 +11,11 @@ set(INDEXBUILDER_FILES - IndexWrapper.cpp + VecIndexCreator.cpp index_c.cpp init_c.cpp + utils.cpp + StringIndexImpl.cpp ) add_library(milvus_indexbuilder SHARED ${INDEXBUILDER_FILES} @@ -27,8 +29,8 @@ endif () # link order matters target_link_libraries(milvus_indexbuilder - knowhere milvus_common + knowhere ${TBB} ${PLATFORM_LIBS} pthread diff --git a/internal/core/src/indexbuilder/IndexCreatorBase.h b/internal/core/src/indexbuilder/IndexCreatorBase.h new file mode 100644 index 0000000000000000000000000000000000000000..5adc4d5a148ff30345560ddcbe8eed24ace44647 --- /dev/null +++ b/internal/core/src/indexbuilder/IndexCreatorBase.h @@ -0,0 +1,39 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License + +#pragma once + +#include "knowhere/common/Dataset.h" +#include "knowhere/common/BinarySet.h" +#include +#include + +namespace milvus::indexbuilder { +class IndexCreatorBase { + public: + virtual ~IndexCreatorBase() = default; + + virtual void + Build(const knowhere::DatasetPtr& dataset) = 0; + + virtual knowhere::BinarySet + Serialize() = 0; + + virtual void + Load(const knowhere::BinarySet&) = 0; + + // virtual knowhere::IndexPtr + // GetIndex() = 0; +}; + +using IndexCreatorBasePtr = std::unique_ptr; + +} // namespace milvus::indexbuilder diff --git a/internal/core/src/indexbuilder/IndexFactory.h b/internal/core/src/indexbuilder/IndexFactory.h new file mode 100644 index 0000000000000000000000000000000000000000..7d7c4aeba7919b656b7508a85058484c588d315b --- /dev/null +++ b/internal/core/src/indexbuilder/IndexFactory.h @@ -0,0 +1,79 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License + +#pragma once + +#include +#include +#include "indexbuilder/IndexCreatorBase.h" +#include "indexbuilder/ScalarIndexCreator.h" +#include "indexbuilder/VecIndexCreator.h" +#include "indexbuilder/type_c.h" +#include +#include + +namespace milvus::indexbuilder { + +// consider template factory if too many factories are needed. +class IndexFactory { + public: + IndexFactory() = default; + IndexFactory(const IndexFactory&) = delete; + IndexFactory + operator=(const IndexFactory&) = delete; + + public: + static IndexFactory& + GetInstance() { + // thread-safe enough after c++ 11 + static IndexFactory instance; + return instance; + } + + IndexCreatorBasePtr + CreateIndex(DataType dtype, const char* type_params, const char* index_params) { + auto real_dtype = proto::schema::DataType(dtype); + auto invalid_dtype_msg = std::string("invalid data type: ") + std::to_string(real_dtype); + + switch (real_dtype) { + case milvus::proto::schema::Bool: + return std::make_unique>(type_params, index_params); + case milvus::proto::schema::Int8: + return std::make_unique>(type_params, index_params); + case milvus::proto::schema::Int16: + return std::make_unique>(type_params, index_params); + case milvus::proto::schema::Int32: + return std::make_unique>(type_params, index_params); + case milvus::proto::schema::Int64: + return std::make_unique>(type_params, index_params); + case milvus::proto::schema::Float: + return std::make_unique>(type_params, index_params); + case milvus::proto::schema::Double: + return std::make_unique>(type_params, index_params); + + case proto::schema::VarChar: + case milvus::proto::schema::String: + return std::make_unique>(type_params, index_params); + + case milvus::proto::schema::BinaryVector: + case milvus::proto::schema::FloatVector: + return std::make_unique(type_params, index_params); + + case milvus::proto::schema::None: + case milvus::proto::schema::DataType_INT_MIN_SENTINEL_DO_NOT_USE_: + case milvus::proto::schema::DataType_INT_MAX_SENTINEL_DO_NOT_USE_: + default: + throw std::invalid_argument(invalid_dtype_msg); + } + } +}; + +} // namespace milvus::indexbuilder diff --git a/internal/core/src/indexbuilder/ScalarIndexCreator-inl.h b/internal/core/src/indexbuilder/ScalarIndexCreator-inl.h new file mode 100644 index 0000000000000000000000000000000000000000..bb986046567cb2fbb764103cd1821f415f1cddb1 --- /dev/null +++ b/internal/core/src/indexbuilder/ScalarIndexCreator-inl.h @@ -0,0 +1,84 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License + +#include +#include +#include "indexbuilder/helper.h" +#include "indexbuilder/StringIndexImpl.h" + +#include +#include +#include + +namespace milvus::indexbuilder { + +template +inline ScalarIndexCreator::ScalarIndexCreator(const char* type_params, const char* index_params) { + // TODO: move parse-related logic to a common interface. + Helper::ParseFromString(type_params_, std::string(type_params)); + Helper::ParseFromString(index_params_, std::string(index_params)); + // TODO: create index according to the params. + index_ = std::make_unique>(); +} + +template +inline void +ScalarIndexCreator::Build(const knowhere::DatasetPtr& dataset) { + auto size = dataset->Get(knowhere::meta::ROWS); + auto data = dataset->Get(knowhere::meta::TENSOR); + index_->Build(size, reinterpret_cast(data)); +} + +template +inline knowhere::BinarySet +ScalarIndexCreator::Serialize() { + return index_->Serialize(config_); +} + +template +inline void +ScalarIndexCreator::Load(const knowhere::BinarySet& binary_set) { + index_->Load(binary_set); +} + +// not sure that the pointer of a golang bool array acts like other types. +template <> +inline void +ScalarIndexCreator::Build(const milvus::knowhere::DatasetPtr& dataset) { + auto size = dataset->Get(knowhere::meta::ROWS); + auto data = dataset->Get(knowhere::meta::TENSOR); + proto::schema::BoolArray arr; + Helper::ParseParams(arr, data, size); + index_->Build(arr.data().size(), arr.data().data()); +} + +template <> +inline ScalarIndexCreator::ScalarIndexCreator(const char* type_params, const char* index_params) { + // TODO: move parse-related logic to a common interface. + Helper::ParseFromString(type_params_, std::string(type_params)); + Helper::ParseFromString(index_params_, std::string(index_params)); + // TODO: create index according to the params. + index_ = std::make_unique(); +} + +template <> +inline void +ScalarIndexCreator::Build(const milvus::knowhere::DatasetPtr& dataset) { + auto size = dataset->Get(knowhere::meta::ROWS); + auto data = dataset->Get(knowhere::meta::TENSOR); + proto::schema::StringArray arr; + Helper::ParseParams(arr, data, size); + // TODO: optimize here. avoid memory copy. + std::vector vecs{arr.data().begin(), arr.data().end()}; + index_->Build(arr.data().size(), vecs.data()); +} + +} // namespace milvus::indexbuilder diff --git a/internal/core/src/indexbuilder/ScalarIndexCreator.h b/internal/core/src/indexbuilder/ScalarIndexCreator.h new file mode 100644 index 0000000000000000000000000000000000000000..7a61d5b5e8c86a09711ad6091c8f181c0350e9dc --- /dev/null +++ b/internal/core/src/indexbuilder/ScalarIndexCreator.h @@ -0,0 +1,49 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License + +#pragma once + +#include "indexbuilder/IndexCreatorBase.h" +#include "knowhere/index/structured_index_simple/StructuredIndex.h" +#include "pb/index_cgo_msg.pb.h" +#include +#include + +namespace milvus::indexbuilder { + +template +class ScalarIndexCreator : public IndexCreatorBase { + // of course, maybe we can support combination index later. + // for example, we can create index for combination of (field a, field b), + // attribute filtering on the combination can be speed up. + static_assert(std::is_fundamental_v || std::is_same_v); + + public: + ScalarIndexCreator(const char* type_params, const char* index_params); + + void + Build(const knowhere::DatasetPtr& dataset) override; + + knowhere::BinarySet + Serialize() override; + + void + Load(const knowhere::BinarySet&) override; + + private: + std::unique_ptr> index_ = nullptr; + proto::indexcgo::TypeParams type_params_; + proto::indexcgo::IndexParams index_params_; + milvus::json config_; +}; +} // namespace milvus::indexbuilder + +#include "ScalarIndexCreator-inl.h" diff --git a/internal/core/src/indexbuilder/StringIndexImpl.cpp b/internal/core/src/indexbuilder/StringIndexImpl.cpp new file mode 100644 index 0000000000000000000000000000000000000000..f97df3e04b5b1193808ecd2f258eb34a4d1a6ab4 --- /dev/null +++ b/internal/core/src/indexbuilder/StringIndexImpl.cpp @@ -0,0 +1,44 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License + +#include "StringIndexImpl.h" + +namespace milvus::indexbuilder { + +// TODO: optimize here. + +knowhere::BinarySet +StringIndexImpl::Serialize(const knowhere::Config& config) { + knowhere::BinarySet res_set; + auto data = this->GetData(); + for (const auto& record : data) { + auto idx = record.idx_; + auto str = record.a_; + std::shared_ptr content(new uint8_t[str.length()]); + memcpy(content.get(), str.c_str(), str.length()); + res_set.Append(std::to_string(idx), content, str.length()); + } + return res_set; +} + +void +StringIndexImpl::Load(const knowhere::BinarySet& index_binary) { + std::vector vecs; + + for (const auto& [k, v] : index_binary.binary_map_) { + std::string str(reinterpret_cast(v->data.get()), v->size); + vecs.emplace_back(str); + } + + Build(vecs.size(), vecs.data()); +} + +} // namespace milvus::indexbuilder diff --git a/internal/core/src/indexbuilder/StringIndexImpl.h b/internal/core/src/indexbuilder/StringIndexImpl.h new file mode 100644 index 0000000000000000000000000000000000000000..55267509bf1ea1c81e62511bb178ec494dbe4684 --- /dev/null +++ b/internal/core/src/indexbuilder/StringIndexImpl.h @@ -0,0 +1,28 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License + +#pragma once + +#include +#include + +namespace milvus::indexbuilder { + +class StringIndexImpl : public knowhere::scalar::StructuredIndexSort { + public: + knowhere::BinarySet + Serialize(const knowhere::Config& config) override; + + void + Load(const knowhere::BinarySet& index_binary) override; +}; + +} // namespace milvus::indexbuilder diff --git a/internal/core/src/indexbuilder/IndexWrapper.cpp b/internal/core/src/indexbuilder/VecIndexCreator.cpp similarity index 74% rename from internal/core/src/indexbuilder/IndexWrapper.cpp rename to internal/core/src/indexbuilder/VecIndexCreator.cpp index 55a0db2a10e1f76db208dff9ae7c33d9fda7bee6..1e4a6d89fffb8628fe9367758997366428bec1d4 100644 --- a/internal/core/src/indexbuilder/IndexWrapper.cpp +++ b/internal/core/src/indexbuilder/VecIndexCreator.cpp @@ -15,7 +15,7 @@ #include "exceptions/EasyAssert.h" #include "pb/index_cgo_msg.pb.h" -#include "indexbuilder/IndexWrapper.h" +#include "indexbuilder/VecIndexCreator.h" #include "indexbuilder/utils.h" #include "knowhere/common/Timer.h" #include "knowhere/common/Utils.h" @@ -25,7 +25,7 @@ namespace milvus::indexbuilder { -IndexWrapper::IndexWrapper(const char* serialized_type_params, const char* serialized_index_params) { +VecIndexCreator::VecIndexCreator(const char* serialized_type_params, const char* serialized_index_params) { type_params_ = std::string(serialized_type_params); index_params_ = std::string(serialized_index_params); @@ -37,18 +37,18 @@ IndexWrapper::IndexWrapper(const char* serialized_type_params, const char* seria AssertInfo(!is_unsupported(index_type, metric_type), index_type + " doesn't support metric: " + metric_type); index_ = knowhere::VecIndexFactory::GetInstance().CreateVecIndex(get_index_type(), index_mode); - AssertInfo(index_ != nullptr, "[IndexWrapper]Index is null after create index"); + AssertInfo(index_ != nullptr, "[VecIndexCreator]Index is null after create index"); } template // ugly here, ParamsT will just be MapParams later void -IndexWrapper::parse_impl(const std::string& serialized_params_str, knowhere::Config& conf) { +VecIndexCreator::parse_impl(const std::string& serialized_params_str, knowhere::Config& conf) { bool deserialized_success; ParamsT params; deserialized_success = google::protobuf::TextFormat::ParseFromString(serialized_params_str, ¶ms); - AssertInfo(deserialized_success, "[IndexWrapper]Deserialize params failed"); + AssertInfo(deserialized_success, "[VecIndexCreator]Deserialize params failed"); for (auto i = 0; i < params.params_size(); ++i) { const auto& param = params.params(i); @@ -112,7 +112,7 @@ IndexWrapper::parse_impl(const std::string& serialized_params_str, knowhere::Con } void -IndexWrapper::parse() { +VecIndexCreator::parse() { namespace indexcgo = milvus::proto::indexcgo; parse_impl(type_params_, type_config_); @@ -124,10 +124,10 @@ IndexWrapper::parse() { template void -IndexWrapper::check_parameter(knowhere::Config& conf, - const std::string& key, - std::function fn, - std::optional default_v) { +VecIndexCreator::check_parameter(knowhere::Config& conf, + const std::string& key, + std::function fn, + std::optional default_v) { if (!conf.contains(key)) { if (default_v.has_value()) { conf[key] = default_v.value(); @@ -140,7 +140,7 @@ IndexWrapper::check_parameter(knowhere::Config& conf, template std::optional -IndexWrapper::get_config_by_name(std::string name) { +VecIndexCreator::get_config_by_name(std::string name) { if (config_.contains(name)) { return {config_[name].get()}; } @@ -148,14 +148,14 @@ IndexWrapper::get_config_by_name(std::string name) { } int64_t -IndexWrapper::dim() { +VecIndexCreator::dim() { auto dimension = get_config_by_name(milvus::knowhere::meta::DIM); - AssertInfo(dimension.has_value(), "[IndexWrapper]Dimension doesn't have value"); + AssertInfo(dimension.has_value(), "[VecIndexCreator]Dimension doesn't have value"); return (dimension.value()); } void -IndexWrapper::BuildWithoutIds(const knowhere::DatasetPtr& dataset) { +VecIndexCreator::BuildWithoutIds(const knowhere::DatasetPtr& dataset) { auto index_type = get_index_type(); auto index_mode = get_index_mode(); config_[knowhere::meta::ROWS] = dataset->Get(knowhere::meta::ROWS); @@ -189,9 +189,9 @@ IndexWrapper::BuildWithoutIds(const knowhere::DatasetPtr& dataset) { } void -IndexWrapper::BuildWithIds(const knowhere::DatasetPtr& dataset) { +VecIndexCreator::BuildWithIds(const knowhere::DatasetPtr& dataset) { AssertInfo(dataset->data().find(milvus::knowhere::meta::IDS) != dataset->data().end(), - "[IndexWrapper]Can't find ids field in dataset"); + "[VecIndexCreator]Can't find ids field in dataset"); auto index_type = get_index_type(); auto index_mode = get_index_mode(); config_[knowhere::meta::ROWS] = dataset->Get(knowhere::meta::ROWS); @@ -212,7 +212,7 @@ IndexWrapper::BuildWithIds(const knowhere::DatasetPtr& dataset) { } void -IndexWrapper::StoreRawData(const knowhere::DatasetPtr& dataset) { +VecIndexCreator::StoreRawData(const knowhere::DatasetPtr& dataset) { auto index_type = get_index_type(); if (is_in_nm_list(index_type)) { auto tensor = dataset->Get(milvus::knowhere::meta::TENSOR); @@ -229,64 +229,25 @@ IndexWrapper::StoreRawData(const knowhere::DatasetPtr& dataset) { } } -std::unique_ptr -IndexWrapper::SerializeBinarySet() { - auto ret = std::make_unique(index_->Serialize(config_)); +milvus::knowhere::BinarySet +VecIndexCreator::Serialize() { + auto ret = index_->Serialize(config_); auto index_type = get_index_type(); if (is_in_nm_list(index_type)) { std::shared_ptr raw_data(new uint8_t[raw_data_.size()], std::default_delete()); memcpy(raw_data.get(), raw_data_.data(), raw_data_.size()); - ret->Append(RAW_DATA, raw_data, raw_data_.size()); + ret.Append(RAW_DATA, raw_data, raw_data_.size()); auto slice_size = get_index_file_slice_size(); // https://github.com/milvus-io/milvus/issues/6421 // Disassemble will only divide the raw vectors, other keys were already divided - knowhere::Disassemble(slice_size * 1024 * 1024, *ret); + knowhere::Disassemble(slice_size * 1024 * 1024, ret); } - return std::move(ret); -} - -/* - * brief Return serialized binary set - * TODO: use a more efficient method to manage memory, consider std::vector later - */ -std::unique_ptr -IndexWrapper::Serialize() { - auto binarySet = index_->Serialize(config_); - auto index_type = get_index_type(); - - if (is_in_nm_list(index_type)) { - std::shared_ptr raw_data(new uint8_t[raw_data_.size()], std::default_delete()); - memcpy(raw_data.get(), raw_data_.data(), raw_data_.size()); - binarySet.Append(RAW_DATA, raw_data, raw_data_.size()); - auto slice_size = get_index_file_slice_size(); - // https://github.com/milvus-io/milvus/issues/6421 - // Disassemble will only divide the raw vectors, other keys were already divided - knowhere::Disassemble(slice_size * 1024 * 1024, binarySet); - } - - namespace indexcgo = milvus::proto::indexcgo; - indexcgo::BinarySet ret; - - for (auto [key, value] : binarySet.binary_map_) { - auto binary = ret.add_datas(); - binary->set_key(key); - binary->set_value(value->data.get(), value->size); - } - - std::string serialized_data; - auto ok = ret.SerializeToString(&serialized_data); - AssertInfo(ok, "[IndexWrapper]Can't serialize data to string"); - - auto binary = std::make_unique(); - binary->data.resize(serialized_data.length()); - memcpy(binary->data.data(), serialized_data.c_str(), serialized_data.length()); - - return binary; + return ret; } void -IndexWrapper::LoadFromBinarySet(milvus::knowhere::BinarySet& binary_set) { +VecIndexCreator::Load(const milvus::knowhere::BinarySet& binary_set) { auto& map_ = binary_set.binary_map_; for (auto it = map_.begin(); it != map_.end(); ++it) { if (it->first == RAW_DATA) { @@ -300,30 +261,8 @@ IndexWrapper::LoadFromBinarySet(milvus::knowhere::BinarySet& binary_set) { index_->Load(binary_set); } -void -IndexWrapper::Load(const char* serialized_sliced_blob_buffer, int32_t size) { - namespace indexcgo = milvus::proto::indexcgo; - auto data = std::string(serialized_sliced_blob_buffer, size); - indexcgo::BinarySet blob_buffer; - - auto ok = blob_buffer.ParseFromString(data); - AssertInfo(ok, "[IndexWrapper]Can't parse data from string to blob_buffer"); - - milvus::knowhere::BinarySet binarySet; - for (auto i = 0; i < blob_buffer.datas_size(); i++) { - const auto& binary = blob_buffer.datas(i); - auto deleter = [&](uint8_t*) {}; // avoid repeated destruction - auto bptr = std::make_shared(); - bptr->data = std::shared_ptr((uint8_t*)binary.value().c_str(), deleter); - bptr->size = binary.value().length(); - binarySet.Append(binary.key(), bptr); - } - - index_->Load(binarySet); -} - std::string -IndexWrapper::get_index_type() { +VecIndexCreator::get_index_type() { // return index_->index_type(); // knowhere bug here // the index_type of all ivf-based index will change to ivf flat after loaded @@ -332,7 +271,7 @@ IndexWrapper::get_index_type() { } std::string -IndexWrapper::get_metric_type() { +VecIndexCreator::get_metric_type() { auto type = get_config_by_name(knowhere::Metric::TYPE); if (type.has_value()) { return type.value(); @@ -347,7 +286,7 @@ IndexWrapper::get_metric_type() { } knowhere::IndexMode -IndexWrapper::get_index_mode() { +VecIndexCreator::get_index_mode() { static std::map mode_map = { {"CPU", knowhere::IndexMode::MODE_CPU}, {"GPU", knowhere::IndexMode::MODE_GPU}, @@ -357,20 +296,20 @@ IndexWrapper::get_index_mode() { } int64_t -IndexWrapper::get_index_file_slice_size() { +VecIndexCreator::get_index_file_slice_size() { if (config_.contains(knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE)) { return config_[knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE].get(); } return 4; // by default } -std::unique_ptr -IndexWrapper::Query(const knowhere::DatasetPtr& dataset) { +std::unique_ptr +VecIndexCreator::Query(const knowhere::DatasetPtr& dataset) { return std::move(QueryImpl(dataset, config_)); } -std::unique_ptr -IndexWrapper::QueryWithParam(const knowhere::DatasetPtr& dataset, const char* serialized_search_params) { +std::unique_ptr +VecIndexCreator::QueryWithParam(const knowhere::DatasetPtr& dataset, const char* serialized_search_params) { namespace indexcgo = milvus::proto::indexcgo; milvus::knowhere::Config search_conf; parse_impl(std::string(serialized_search_params), search_conf); @@ -378,8 +317,8 @@ IndexWrapper::QueryWithParam(const knowhere::DatasetPtr& dataset, const char* se return std::move(QueryImpl(dataset, search_conf)); } -std::unique_ptr -IndexWrapper::QueryImpl(const knowhere::DatasetPtr& dataset, const knowhere::Config& conf) { +std::unique_ptr +VecIndexCreator::QueryImpl(const knowhere::DatasetPtr& dataset, const knowhere::Config& conf) { auto load_raw_data_closure = [&]() { LoadRawData(); }; // hide this pointer auto index_type = get_index_type(); if (is_in_nm_list(index_type)) { @@ -392,7 +331,7 @@ IndexWrapper::QueryImpl(const knowhere::DatasetPtr& dataset, const knowhere::Con auto nq = dataset->Get(milvus::knowhere::meta::ROWS); auto k = config_[milvus::knowhere::meta::TOPK].get(); - auto query_res = std::make_unique(); + auto query_res = std::make_unique(); query_res->nq = nq; query_res->topk = k; query_res->ids.resize(nq * k); @@ -404,7 +343,7 @@ IndexWrapper::QueryImpl(const knowhere::DatasetPtr& dataset, const knowhere::Con } void -IndexWrapper::LoadRawData() { +VecIndexCreator::LoadRawData() { auto index_type = get_index_type(); if (is_in_nm_list(index_type)) { auto bs = index_->Serialize(config_); diff --git a/internal/core/src/indexbuilder/IndexWrapper.h b/internal/core/src/indexbuilder/VecIndexCreator.h similarity index 84% rename from internal/core/src/indexbuilder/IndexWrapper.h rename to internal/core/src/indexbuilder/VecIndexCreator.h index bb1f285d600ee5d8234e22f8a9d54e2e647629cd..63b7d37da9b7d1521b8201382f8484b07942c454 100644 --- a/internal/core/src/indexbuilder/IndexWrapper.h +++ b/internal/core/src/indexbuilder/VecIndexCreator.h @@ -18,35 +18,31 @@ #include "knowhere/index/vector_index/VecIndex.h" #include "knowhere/common/BinarySet.h" +#include "indexbuilder/IndexCreatorBase.h" namespace milvus::indexbuilder { -class IndexWrapper { +// TODO: better to distinguish binary vec & float vec. +class VecIndexCreator : public IndexCreatorBase { public: - explicit IndexWrapper(const char* serialized_type_params, const char* serialized_index_params); - - int64_t - dim(); + explicit VecIndexCreator(const char* serialized_type_params, const char* serialized_index_params); void - BuildWithoutIds(const knowhere::DatasetPtr& dataset); - - struct Binary { - std::vector data; - }; - - std::unique_ptr - Serialize(); + Build(const knowhere::DatasetPtr& dataset) override { + BuildWithoutIds(dataset); + } - std::unique_ptr - SerializeBinarySet(); + knowhere::BinarySet + Serialize() override; void - LoadFromBinarySet(milvus::knowhere::BinarySet&); + Load(const knowhere::BinarySet& binary_set) override; - void - Load(const char* serialized_sliced_blob_buffer, int32_t size); + int64_t + dim(); + public: + // used for tests struct QueryResult { std::vector ids; std::vector distances; @@ -104,6 +100,9 @@ class IndexWrapper { void BuildWithIds(const knowhere::DatasetPtr& dataset); + void + BuildWithoutIds(const knowhere::DatasetPtr& dataset); + private: knowhere::VecIndexPtr index_ = nullptr; std::string type_params_; diff --git a/internal/core/src/indexbuilder/helper.h b/internal/core/src/indexbuilder/helper.h new file mode 100644 index 0000000000000000000000000000000000000000..dd8efba40ba8c96020c04973ea2d818ad2ff0ffb --- /dev/null +++ b/internal/core/src/indexbuilder/helper.h @@ -0,0 +1,38 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License + +#pragma once + +#include "pb/index_cgo_msg.pb.h" +#include "exceptions/EasyAssert.h" +#include +#include +#include + +namespace milvus::indexbuilder { + +using MapParams = std::map; + +struct Helper { + static void + ParseFromString(google::protobuf::Message& params, const std::string& str) { + auto ok = google::protobuf::TextFormat::ParseFromString(str, ¶ms); + AssertInfo(ok, "failed to parse params from string"); + } + + static void + ParseParams(google::protobuf::Message& params, const void* data, const size_t size) { + auto ok = params.ParseFromArray(data, size); + AssertInfo(ok, "failed to parse params from array"); + } +}; + +} // namespace milvus::indexbuilder diff --git a/internal/core/src/indexbuilder/index_c.cpp b/internal/core/src/indexbuilder/index_c.cpp index b6d99d0e176b0f3f550da67a9bd7eff307fbcc42..0f769eb556ab59061f3583a0eb9e30a25330d15a 100644 --- a/internal/core/src/indexbuilder/index_c.cpp +++ b/internal/core/src/indexbuilder/index_c.cpp @@ -17,28 +17,23 @@ #endif +#include "exceptions/EasyAssert.h" #include "knowhere/index/vector_index/adapter/VectorAdapter.h" -#include "indexbuilder/IndexWrapper.h" +#include "indexbuilder/VecIndexCreator.h" #include "indexbuilder/index_c.h" - -class CGODebugUtils { - public: - static int64_t - Strlen(const char* str, int64_t size) { - if (size == 0) { - return size; - } else { - return strlen(str); - } - } -}; +#include "indexbuilder/IndexFactory.h" CStatus -CreateIndex(const char* serialized_type_params, const char* serialized_index_params, CIndex* res_index) { +CreateIndex(DataType dtype, + const char* serialized_type_params, + const char* serialized_index_params, + CIndex* res_index) { auto status = CStatus(); try { - auto index = - std::make_unique(serialized_type_params, serialized_index_params); + AssertInfo(res_index, "failed to create index, passed index was null"); + + auto index = milvus::indexbuilder::IndexFactory::GetInstance().CreateIndex(dtype, serialized_type_params, + serialized_index_params); *res_index = index.release(); status.error_code = Success; status.error_msg = ""; @@ -49,24 +44,16 @@ CreateIndex(const char* serialized_type_params, const char* serialized_index_par return status; } -void -DeleteIndex(CIndex index) { - auto cIndex = (milvus::indexbuilder::IndexWrapper*)index; - delete cIndex; -#ifdef __linux__ - malloc_trim(0); -#endif -} - CStatus -BuildFloatVecIndexWithoutIds(CIndex index, int64_t float_value_num, const float* vectors) { +DeleteIndex(CIndex index) { auto status = CStatus(); try { - auto cIndex = (milvus::indexbuilder::IndexWrapper*)index; - auto dim = cIndex->dim(); - auto row_nums = float_value_num / dim; - auto ds = milvus::knowhere::GenDataset(row_nums, dim, vectors); - cIndex->BuildWithoutIds(ds); + AssertInfo(index, "failed to delete index, passed index was null"); + auto cIndex = reinterpret_cast(index); + delete cIndex; +#ifdef __linux__ + malloc_trim(0); +#endif status.error_code = Success; status.error_msg = ""; } catch (std::exception& e) { @@ -77,14 +64,16 @@ BuildFloatVecIndexWithoutIds(CIndex index, int64_t float_value_num, const float* } CStatus -BuildBinaryVecIndexWithoutIds(CIndex index, int64_t data_size, const uint8_t* vectors) { +BuildFloatVecIndex(CIndex index, int64_t float_value_num, const float* vectors) { auto status = CStatus(); try { - auto cIndex = (milvus::indexbuilder::IndexWrapper*)index; + AssertInfo(index, "failed to build float vector index, passed index was null"); + auto real_index = reinterpret_cast(index); + auto cIndex = dynamic_cast(real_index); auto dim = cIndex->dim(); - auto row_nums = (data_size * 8) / dim; + auto row_nums = float_value_num / dim; auto ds = milvus::knowhere::GenDataset(row_nums, dim, vectors); - cIndex->BuildWithoutIds(ds); + cIndex->Build(ds); status.error_code = Success; status.error_msg = ""; } catch (std::exception& e) { @@ -95,12 +84,16 @@ BuildBinaryVecIndexWithoutIds(CIndex index, int64_t data_size, const uint8_t* ve } CStatus -SerializeToBinarySet(CIndex index, CBinarySet* c_binary_set) { +BuildBinaryVecIndex(CIndex index, int64_t data_size, const uint8_t* vectors) { auto status = CStatus(); try { - auto cIndex = (milvus::indexbuilder::IndexWrapper*)index; - auto binary = cIndex->SerializeBinarySet(); - *c_binary_set = binary.release(); + AssertInfo(index, "failed to build binary vector index, passed index was null"); + auto real_index = reinterpret_cast(index); + auto cIndex = dynamic_cast(real_index); + auto dim = cIndex->dim(); + auto row_nums = (data_size * 8) / dim; + auto ds = milvus::knowhere::GenDataset(row_nums, dim, vectors); + cIndex->Build(ds); status.error_code = Success; status.error_msg = ""; } catch (std::exception& e) { @@ -110,13 +103,22 @@ SerializeToBinarySet(CIndex index, CBinarySet* c_binary_set) { return status; } +// field_data: +// 1, serialized proto::schema::BoolArray, if type is bool; +// 2, serialized proto::schema::StringArray, if type is string; +// 3, raw pointer, if type is of fundamental except bool type; +// TODO: optimize here if necessary. CStatus -SerializeToSlicedBuffer(CIndex index, CBinary* c_binary) { +BuildScalarIndex(CIndex c_index, int64_t size, const void* field_data) { auto status = CStatus(); try { - auto cIndex = (milvus::indexbuilder::IndexWrapper*)index; - auto binary = cIndex->Serialize(); - *c_binary = binary.release(); + AssertInfo(c_index, "failed to build scalar index, passed index was null"); + + auto real_index = reinterpret_cast(c_index); + const int64_t dim = 8; // not important here + auto dataset = milvus::knowhere::GenDataset(size, dim, field_data); + real_index->Build(dataset); + status.error_code = Success; status.error_msg = ""; } catch (std::exception& e) { @@ -126,31 +128,14 @@ SerializeToSlicedBuffer(CIndex index, CBinary* c_binary) { return status; } -int64_t -GetCBinarySize(CBinary c_binary) { - auto cBinary = (milvus::indexbuilder::IndexWrapper::Binary*)c_binary; - return cBinary->data.size(); -} - -// Note: the memory of data has been allocated outside -void -GetCBinaryData(CBinary c_binary, void* data) { - auto cBinary = (milvus::indexbuilder::IndexWrapper::Binary*)c_binary; - memcpy(data, cBinary->data.data(), cBinary->data.size()); -} - -void -DeleteCBinary(CBinary c_binary) { - auto cBinary = (milvus::indexbuilder::IndexWrapper::Binary*)c_binary; - delete cBinary; -} - CStatus -LoadFromSlicedBuffer(CIndex index, const char* serialized_sliced_blob_buffer, int32_t size) { +SerializeIndexToBinarySet(CIndex index, CBinarySet* c_binary_set) { auto status = CStatus(); try { - auto cIndex = (milvus::indexbuilder::IndexWrapper*)index; - cIndex->Load(serialized_sliced_blob_buffer, size); + AssertInfo(index, "failed to serialize index to binary set, passed index was null"); + auto real_index = reinterpret_cast(index); + auto binary = std::make_unique(real_index->Serialize()); + *c_binary_set = binary.release(); status.error_code = Success; status.error_msg = ""; } catch (std::exception& e) { @@ -161,12 +146,13 @@ LoadFromSlicedBuffer(CIndex index, const char* serialized_sliced_blob_buffer, in } CStatus -LoadFromBinarySet(CIndex index, CBinarySet c_binary_set) { +LoadIndexFromBinarySet(CIndex index, CBinarySet c_binary_set) { auto status = CStatus(); try { - auto cIndex = (milvus::indexbuilder::IndexWrapper*)index; - auto binary_set = (milvus::knowhere::BinarySet*)c_binary_set; - cIndex->LoadFromBinarySet(*binary_set); + AssertInfo(index, "failed to load index from binary set, passed index was null"); + auto real_index = reinterpret_cast(index); + auto binary_set = reinterpret_cast(c_binary_set); + real_index->Load(*binary_set); status.error_code = Success; status.error_msg = ""; } catch (std::exception& e) { @@ -180,7 +166,7 @@ CStatus QueryOnFloatVecIndex(CIndex index, int64_t float_value_num, const float* vectors, CIndexQueryResult* res) { auto status = CStatus(); try { - auto cIndex = (milvus::indexbuilder::IndexWrapper*)index; + auto cIndex = (milvus::indexbuilder::VecIndexCreator*)index; auto dim = cIndex->dim(); auto row_nums = float_value_num / dim; auto query_ds = milvus::knowhere::GenDataset(row_nums, dim, vectors); @@ -204,7 +190,7 @@ QueryOnFloatVecIndexWithParam(CIndex index, CIndexQueryResult* res) { auto status = CStatus(); try { - auto cIndex = (milvus::indexbuilder::IndexWrapper*)index; + auto cIndex = (milvus::indexbuilder::VecIndexCreator*)index; auto dim = cIndex->dim(); auto row_nums = float_value_num / dim; auto query_ds = milvus::knowhere::GenDataset(row_nums, dim, vectors); @@ -224,7 +210,7 @@ CStatus QueryOnBinaryVecIndex(CIndex index, int64_t data_size, const uint8_t* vectors, CIndexQueryResult* res) { auto status = CStatus(); try { - auto cIndex = (milvus::indexbuilder::IndexWrapper*)index; + auto cIndex = (milvus::indexbuilder::VecIndexCreator*)index; auto dim = cIndex->dim(); auto row_nums = (data_size * 8) / dim; auto query_ds = milvus::knowhere::GenDataset(row_nums, dim, vectors); @@ -248,7 +234,7 @@ QueryOnBinaryVecIndexWithParam(CIndex index, CIndexQueryResult* res) { auto status = CStatus(); try { - auto cIndex = (milvus::indexbuilder::IndexWrapper*)index; + auto cIndex = (milvus::indexbuilder::VecIndexCreator*)index; auto dim = cIndex->dim(); auto row_nums = (data_size * 8) / dim; auto query_ds = milvus::knowhere::GenDataset(row_nums, dim, vectors); @@ -268,7 +254,7 @@ CStatus CreateQueryResult(CIndexQueryResult* res) { auto status = CStatus(); try { - auto query_result = std::make_unique(); + auto query_result = std::make_unique(); *res = query_result.release(); status.error_code = Success; @@ -282,19 +268,19 @@ CreateQueryResult(CIndexQueryResult* res) { int64_t NqOfQueryResult(CIndexQueryResult res) { - auto c_res = (milvus::indexbuilder::IndexWrapper::QueryResult*)res; + auto c_res = (milvus::indexbuilder::VecIndexCreator::QueryResult*)res; return c_res->nq; } int64_t TopkOfQueryResult(CIndexQueryResult res) { - auto c_res = (milvus::indexbuilder::IndexWrapper::QueryResult*)res; + auto c_res = (milvus::indexbuilder::VecIndexCreator::QueryResult*)res; return c_res->topk; } void GetIdsOfQueryResult(CIndexQueryResult res, int64_t* ids) { - auto c_res = (milvus::indexbuilder::IndexWrapper::QueryResult*)res; + auto c_res = (milvus::indexbuilder::VecIndexCreator::QueryResult*)res; auto nq = c_res->nq; auto k = c_res->topk; // TODO: how could we avoid memory copy whenever this called @@ -303,7 +289,7 @@ GetIdsOfQueryResult(CIndexQueryResult res, int64_t* ids) { void GetDistancesOfQueryResult(CIndexQueryResult res, float* distances) { - auto c_res = (milvus::indexbuilder::IndexWrapper::QueryResult*)res; + auto c_res = (milvus::indexbuilder::VecIndexCreator::QueryResult*)res; auto nq = c_res->nq; auto k = c_res->topk; // TODO: how could we avoid memory copy whenever this called @@ -314,7 +300,7 @@ CStatus DeleteIndexQueryResult(CIndexQueryResult res) { auto status = CStatus(); try { - auto c_res = (milvus::indexbuilder::IndexWrapper::QueryResult*)res; + auto c_res = (milvus::indexbuilder::VecIndexCreator::QueryResult*)res; delete c_res; status.error_code = Success; @@ -325,8 +311,3 @@ DeleteIndexQueryResult(CIndexQueryResult res) { } return status; } - -void -DeleteByteArray(const char* array) { - delete[] array; -} diff --git a/internal/core/src/indexbuilder/index_c.h b/internal/core/src/indexbuilder/index_c.h index 2a312f00344a9e7e94566fa0d3c6abf1c6bedfe3..3971ba38035b1c0999f38c112bd20ff1daef8172 100644 --- a/internal/core/src/indexbuilder/index_c.h +++ b/internal/core/src/indexbuilder/index_c.h @@ -16,50 +16,38 @@ extern "C" { #endif #include -#include "segcore/collection_c.h" #include "common/type_c.h" #include "common/vector_index_c.h" - -typedef void* CIndex; -typedef void* CIndexQueryResult; -typedef void* CBinary; - -// TODO: how could we pass map between go and c++ more efficiently? -// Solution: using Protobuf instead of JSON, this way significantly increase programming efficiency +#include "indexbuilder/type_c.h" CStatus -CreateIndex(const char* serialized_type_params, const char* serialized_index_params, CIndex* res_index); - -void -DeleteIndex(CIndex index); +CreateIndex(enum DataType dtype, + const char* serialized_type_params, + const char* serialized_index_params, + CIndex* res_index); CStatus -BuildFloatVecIndexWithoutIds(CIndex index, int64_t float_value_num, const float* vectors); +DeleteIndex(CIndex index); CStatus -BuildBinaryVecIndexWithoutIds(CIndex index, int64_t data_size, const uint8_t* vectors); +BuildFloatVecIndex(CIndex index, int64_t float_value_num, const float* vectors); CStatus -SerializeToSlicedBuffer(CIndex index, CBinary* c_binary); +BuildBinaryVecIndex(CIndex index, int64_t data_size, const uint8_t* vectors); +// field_data: +// 1, serialized proto::schema::BoolArray, if type is bool; +// 2, serialized proto::schema::StringArray, if type is string; +// 3, raw pointer, if type is of fundamental except bool type; +// TODO: optimize here if necessary. CStatus -SerializeToBinarySet(CIndex index, CBinarySet* c_binary_set); - -int64_t -GetCBinarySize(CBinary c_binary); - -// Note: the memory of data is allocated outside -void -GetCBinaryData(CBinary c_binary, void* data); - -void -DeleteCBinary(CBinary c_binary); +BuildScalarIndex(CIndex c_index, int64_t size, const void* field_data); CStatus -LoadFromSlicedBuffer(CIndex index, const char* serialized_sliced_blob_buffer, int32_t size); +SerializeIndexToBinarySet(CIndex index, CBinarySet* c_binary_set); CStatus -LoadFromBinarySet(CIndex index, CBinarySet c_binary_set); +LoadIndexFromBinarySet(CIndex index, CBinarySet c_binary_set); CStatus QueryOnFloatVecIndex(CIndex index, int64_t float_value_num, const float* vectors, CIndexQueryResult* res); @@ -99,9 +87,6 @@ GetDistancesOfQueryResult(CIndexQueryResult res, float* distances); CStatus DeleteIndexQueryResult(CIndexQueryResult res); -void -DeleteByteArray(const char* array); - #ifdef __cplusplus }; #endif diff --git a/internal/core/src/indexbuilder/type_c.h b/internal/core/src/indexbuilder/type_c.h new file mode 100644 index 0000000000000000000000000000000000000000..a60a4ee92bc881152182fd7c7988d59b6330a025 --- /dev/null +++ b/internal/core/src/indexbuilder/type_c.h @@ -0,0 +1,35 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License + +#pragma once + +// pure C don't support that we use schemapb.DataType directly. +// Note: the value of all enumerations must match the corresponding schemapb.DataType. +// TODO: what if there are increments in schemapb.DataType. +enum DataType { + None = 0, + Bool = 1, + Int8 = 2, + Int16 = 3, + Int32 = 4, + Int64 = 5, + + Float = 10, + Double = 11, + + String = 20, + + BinaryVector = 100, + FloatVector = 101, +}; + +typedef void* CIndex; +typedef void* CIndexQueryResult; diff --git a/internal/core/src/indexbuilder/utils.cpp b/internal/core/src/indexbuilder/utils.cpp new file mode 100644 index 0000000000000000000000000000000000000000..2b576d2904c0f0f5ac0e97d59bd3e7d0695f0885 --- /dev/null +++ b/internal/core/src/indexbuilder/utils.cpp @@ -0,0 +1,102 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License + +#include "indexbuilder/utils.h" + +#include +#include +#include +#include +#include + +#include "knowhere/index/IndexType.h" + +namespace milvus::indexbuilder { + +std::vector +NM_List() { + static std::vector ret{ + milvus::knowhere::IndexEnum::INDEX_FAISS_IVFFLAT, + milvus::knowhere::IndexEnum::INDEX_NSG, + milvus::knowhere::IndexEnum::INDEX_RHNSWFlat, + }; + return ret; +} + +std::vector +BIN_List() { + static std::vector ret{ + milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP, + milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, + }; + return ret; +} + +std::vector +Need_ID_List() { + static std::vector ret{ + // milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, + // milvus::knowhere::IndexEnum::INDEX_NSG, + }; + + return ret; +} + +std::vector +Need_BuildAll_list() { + static std::vector ret{ + milvus::knowhere::IndexEnum::INDEX_NSG, + }; + return ret; +} + +std::vector> +unsupported_index_combinations() { + static std::vector> ret{ + std::make_tuple(std::string(knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT), std::string(knowhere::Metric::L2)), + }; + return ret; +} + +template +bool +is_in_list(const T& t, std::function()> list_func) { + auto l = list_func(); + return std::find(l.begin(), l.end(), t) != l.end(); +} + +bool +is_in_bin_list(const milvus::knowhere::IndexType& index_type) { + return is_in_list(index_type, BIN_List); +} + +bool +is_in_nm_list(const milvus::knowhere::IndexType& index_type) { + return is_in_list(index_type, NM_List); +} + +bool +is_in_need_build_all_list(const milvus::knowhere::IndexType& index_type) { + return is_in_list(index_type, Need_BuildAll_list); +} + +bool +is_in_need_id_list(const milvus::knowhere::IndexType& index_type) { + return is_in_list(index_type, Need_ID_List); +} + +bool +is_unsupported(const milvus::knowhere::IndexType& index_type, const milvus::knowhere::MetricType& metric_type) { + return is_in_list>(std::make_tuple(index_type, metric_type), + unsupported_index_combinations); +} + +} // namespace milvus::indexbuilder diff --git a/internal/core/src/indexbuilder/utils.h b/internal/core/src/indexbuilder/utils.h index ea15b035dcf25255b6972ae72aada24918313448..ca59c14bcdf10158024c95822347cf352b6ba951 100644 --- a/internal/core/src/indexbuilder/utils.h +++ b/internal/core/src/indexbuilder/utils.h @@ -15,87 +15,45 @@ #include #include #include +#include +#include #include "knowhere/index/IndexType.h" namespace milvus::indexbuilder { std::vector -NM_List() { - static std::vector ret{ - milvus::knowhere::IndexEnum::INDEX_FAISS_IVFFLAT, - milvus::knowhere::IndexEnum::INDEX_NSG, - milvus::knowhere::IndexEnum::INDEX_RHNSWFlat, - }; - return ret; -} +NM_List(); std::vector -BIN_List() { - static std::vector ret{ - milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP, - milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, - }; - return ret; -} +BIN_List(); std::vector -Need_ID_List() { - static std::vector ret{ - // milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, - // milvus::knowhere::IndexEnum::INDEX_NSG, - }; - - return ret; -} +Need_ID_List(); std::vector -Need_BuildAll_list() { - static std::vector ret{ - milvus::knowhere::IndexEnum::INDEX_NSG, - }; - return ret; -} +Need_BuildAll_list(); std::vector> -unsupported_index_combinations() { - static std::vector> ret{ - std::make_tuple(std::string(knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT), std::string(knowhere::Metric::L2)), - }; - return ret; -} +unsupported_index_combinations(); template bool -is_in_list(const T& t, std::function()> list_func) { - auto l = list_func(); - return std::find(l.begin(), l.end(), t) != l.end(); -} +is_in_list(const T& t, std::function()> list_func); bool -is_in_bin_list(const milvus::knowhere::IndexType& index_type) { - return is_in_list(index_type, BIN_List); -} +is_in_bin_list(const milvus::knowhere::IndexType& index_type); bool -is_in_nm_list(const milvus::knowhere::IndexType& index_type) { - return is_in_list(index_type, NM_List); -} +is_in_nm_list(const milvus::knowhere::IndexType& index_type); bool -is_in_need_build_all_list(const milvus::knowhere::IndexType& index_type) { - return is_in_list(index_type, Need_BuildAll_list); -} +is_in_need_build_all_list(const milvus::knowhere::IndexType& index_type); bool -is_in_need_id_list(const milvus::knowhere::IndexType& index_type) { - return is_in_list(index_type, Need_ID_List); -} +is_in_need_id_list(const milvus::knowhere::IndexType& index_type); bool -is_unsupported(const milvus::knowhere::IndexType& index_type, const milvus::knowhere::MetricType& metric_type) { - return is_in_list>(std::make_tuple(index_type, metric_type), - unsupported_index_combinations); -} +is_unsupported(const milvus::knowhere::IndexType& index_type, const milvus::knowhere::MetricType& metric_type); } // namespace milvus::indexbuilder diff --git a/internal/core/src/pb/index_cgo_msg.pb.cc b/internal/core/src/pb/index_cgo_msg.pb.cc index 5d9f5141e633e74b1c008fd4c6f338cf5eed17dc..96904039e865bea8278bb58fc7b68fd66e11f336 100644 --- a/internal/core/src/pb/index_cgo_msg.pb.cc +++ b/internal/core/src/pb/index_cgo_msg.pb.cc @@ -17,6 +17,7 @@ #include extern PROTOBUF_INTERNAL_EXPORT_common_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_KeyValuePair_common_2eproto; extern PROTOBUF_INTERNAL_EXPORT_index_5fcgo_5fmsg_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_Binary_index_5fcgo_5fmsg_2eproto; +extern PROTOBUF_INTERNAL_EXPORT_index_5fcgo_5fmsg_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_MapParamsV2_ParamsEntry_DoNotUse_index_5fcgo_5fmsg_2eproto; namespace milvus { namespace proto { namespace indexcgo { @@ -32,6 +33,14 @@ class MapParamsDefaultTypeInternal { public: ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; } _MapParams_default_instance_; +class MapParamsV2_ParamsEntry_DoNotUseDefaultTypeInternal { + public: + ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; +} _MapParamsV2_ParamsEntry_DoNotUse_default_instance_; +class MapParamsV2DefaultTypeInternal { + public: + ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; +} _MapParamsV2_default_instance_; class BinaryDefaultTypeInternal { public: ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; @@ -102,6 +111,34 @@ static void InitDefaultsscc_info_MapParams_index_5fcgo_5fmsg_2eproto() { {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 1, InitDefaultsscc_info_MapParams_index_5fcgo_5fmsg_2eproto}, { &scc_info_KeyValuePair_common_2eproto.base,}}; +static void InitDefaultsscc_info_MapParamsV2_index_5fcgo_5fmsg_2eproto() { + GOOGLE_PROTOBUF_VERIFY_VERSION; + + { + void* ptr = &::milvus::proto::indexcgo::_MapParamsV2_default_instance_; + new (ptr) ::milvus::proto::indexcgo::MapParamsV2(); + ::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyMessage(ptr); + } + ::milvus::proto::indexcgo::MapParamsV2::InitAsDefaultInstance(); +} + +::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<1> scc_info_MapParamsV2_index_5fcgo_5fmsg_2eproto = + {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 1, InitDefaultsscc_info_MapParamsV2_index_5fcgo_5fmsg_2eproto}, { + &scc_info_MapParamsV2_ParamsEntry_DoNotUse_index_5fcgo_5fmsg_2eproto.base,}}; + +static void InitDefaultsscc_info_MapParamsV2_ParamsEntry_DoNotUse_index_5fcgo_5fmsg_2eproto() { + GOOGLE_PROTOBUF_VERIFY_VERSION; + + { + void* ptr = &::milvus::proto::indexcgo::_MapParamsV2_ParamsEntry_DoNotUse_default_instance_; + new (ptr) ::milvus::proto::indexcgo::MapParamsV2_ParamsEntry_DoNotUse(); + } + ::milvus::proto::indexcgo::MapParamsV2_ParamsEntry_DoNotUse::InitAsDefaultInstance(); +} + +::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_MapParamsV2_ParamsEntry_DoNotUse_index_5fcgo_5fmsg_2eproto = + {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 0, InitDefaultsscc_info_MapParamsV2_ParamsEntry_DoNotUse_index_5fcgo_5fmsg_2eproto}, {}}; + static void InitDefaultsscc_info_TypeParams_index_5fcgo_5fmsg_2eproto() { GOOGLE_PROTOBUF_VERIFY_VERSION; @@ -117,7 +154,7 @@ static void InitDefaultsscc_info_TypeParams_index_5fcgo_5fmsg_2eproto() { {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 1, InitDefaultsscc_info_TypeParams_index_5fcgo_5fmsg_2eproto}, { &scc_info_KeyValuePair_common_2eproto.base,}}; -static ::PROTOBUF_NAMESPACE_ID::Metadata file_level_metadata_index_5fcgo_5fmsg_2eproto[5]; +static ::PROTOBUF_NAMESPACE_ID::Metadata file_level_metadata_index_5fcgo_5fmsg_2eproto[7]; static constexpr ::PROTOBUF_NAMESPACE_ID::EnumDescriptor const** file_level_enum_descriptors_index_5fcgo_5fmsg_2eproto = nullptr; static constexpr ::PROTOBUF_NAMESPACE_ID::ServiceDescriptor const** file_level_service_descriptors_index_5fcgo_5fmsg_2eproto = nullptr; @@ -140,6 +177,21 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_index_5fcgo_5fmsg_2eproto::off ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ PROTOBUF_FIELD_OFFSET(::milvus::proto::indexcgo::MapParams, params_), + PROTOBUF_FIELD_OFFSET(::milvus::proto::indexcgo::MapParamsV2_ParamsEntry_DoNotUse, _has_bits_), + PROTOBUF_FIELD_OFFSET(::milvus::proto::indexcgo::MapParamsV2_ParamsEntry_DoNotUse, _internal_metadata_), + ~0u, // no _extensions_ + ~0u, // no _oneof_case_ + ~0u, // no _weak_field_map_ + PROTOBUF_FIELD_OFFSET(::milvus::proto::indexcgo::MapParamsV2_ParamsEntry_DoNotUse, key_), + PROTOBUF_FIELD_OFFSET(::milvus::proto::indexcgo::MapParamsV2_ParamsEntry_DoNotUse, value_), + 0, + 1, + ~0u, // no _has_bits_ + PROTOBUF_FIELD_OFFSET(::milvus::proto::indexcgo::MapParamsV2, _internal_metadata_), + ~0u, // no _extensions_ + ~0u, // no _oneof_case_ + ~0u, // no _weak_field_map_ + PROTOBUF_FIELD_OFFSET(::milvus::proto::indexcgo::MapParamsV2, params_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::milvus::proto::indexcgo::Binary, _internal_metadata_), ~0u, // no _extensions_ @@ -158,14 +210,18 @@ static const ::PROTOBUF_NAMESPACE_ID::internal::MigrationSchema schemas[] PROTOB { 0, -1, sizeof(::milvus::proto::indexcgo::TypeParams)}, { 6, -1, sizeof(::milvus::proto::indexcgo::IndexParams)}, { 12, -1, sizeof(::milvus::proto::indexcgo::MapParams)}, - { 18, -1, sizeof(::milvus::proto::indexcgo::Binary)}, - { 25, -1, sizeof(::milvus::proto::indexcgo::BinarySet)}, + { 18, 25, sizeof(::milvus::proto::indexcgo::MapParamsV2_ParamsEntry_DoNotUse)}, + { 27, -1, sizeof(::milvus::proto::indexcgo::MapParamsV2)}, + { 33, -1, sizeof(::milvus::proto::indexcgo::Binary)}, + { 40, -1, sizeof(::milvus::proto::indexcgo::BinarySet)}, }; static ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] = { reinterpret_cast(&::milvus::proto::indexcgo::_TypeParams_default_instance_), reinterpret_cast(&::milvus::proto::indexcgo::_IndexParams_default_instance_), reinterpret_cast(&::milvus::proto::indexcgo::_MapParams_default_instance_), + reinterpret_cast(&::milvus::proto::indexcgo::_MapParamsV2_ParamsEntry_DoNotUse_default_instance_), + reinterpret_cast(&::milvus::proto::indexcgo::_MapParamsV2_default_instance_), reinterpret_cast(&::milvus::proto::indexcgo::_Binary_default_instance_), reinterpret_cast(&::milvus::proto::indexcgo::_BinarySet_default_instance_), }; @@ -177,29 +233,34 @@ const char descriptor_table_protodef_index_5fcgo_5fmsg_2eproto[] PROTOBUF_SECTIO "air\"@\n\013IndexParams\0221\n\006params\030\001 \003(\0132!.mil" "vus.proto.common.KeyValuePair\">\n\tMapPara" "ms\0221\n\006params\030\001 \003(\0132!.milvus.proto.common" - ".KeyValuePair\"$\n\006Binary\022\013\n\003key\030\001 \001(\t\022\r\n\005" - "value\030\002 \001(\014\"9\n\tBinarySet\022,\n\005datas\030\001 \003(\0132" - "\035.milvus.proto.indexcgo.BinaryB7Z5github" - ".com/milvus-io/milvus/internal/proto/ind" - "excgopbb\006proto3" + ".KeyValuePair\"|\n\013MapParamsV2\022>\n\006params\030\001" + " \003(\0132..milvus.proto.indexcgo.MapParamsV2" + ".ParamsEntry\032-\n\013ParamsEntry\022\013\n\003key\030\001 \001(\t" + "\022\r\n\005value\030\002 \001(\t:\0028\001\"$\n\006Binary\022\013\n\003key\030\001 \001" + "(\t\022\r\n\005value\030\002 \001(\014\"9\n\tBinarySet\022,\n\005datas\030" + "\001 \003(\0132\035.milvus.proto.indexcgo.BinaryB7Z5" + "github.com/milvus-io/milvus/internal/pro" + "to/indexcgopbb\006proto3" ; static const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable*const descriptor_table_index_5fcgo_5fmsg_2eproto_deps[1] = { &::descriptor_table_common_2eproto, }; -static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_index_5fcgo_5fmsg_2eproto_sccs[5] = { +static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_index_5fcgo_5fmsg_2eproto_sccs[7] = { &scc_info_Binary_index_5fcgo_5fmsg_2eproto.base, &scc_info_BinarySet_index_5fcgo_5fmsg_2eproto.base, &scc_info_IndexParams_index_5fcgo_5fmsg_2eproto.base, &scc_info_MapParams_index_5fcgo_5fmsg_2eproto.base, + &scc_info_MapParamsV2_index_5fcgo_5fmsg_2eproto.base, + &scc_info_MapParamsV2_ParamsEntry_DoNotUse_index_5fcgo_5fmsg_2eproto.base, &scc_info_TypeParams_index_5fcgo_5fmsg_2eproto.base, }; static ::PROTOBUF_NAMESPACE_ID::internal::once_flag descriptor_table_index_5fcgo_5fmsg_2eproto_once; static bool descriptor_table_index_5fcgo_5fmsg_2eproto_initialized = false; const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable descriptor_table_index_5fcgo_5fmsg_2eproto = { - &descriptor_table_index_5fcgo_5fmsg_2eproto_initialized, descriptor_table_protodef_index_5fcgo_5fmsg_2eproto, "index_cgo_msg.proto", 415, - &descriptor_table_index_5fcgo_5fmsg_2eproto_once, descriptor_table_index_5fcgo_5fmsg_2eproto_sccs, descriptor_table_index_5fcgo_5fmsg_2eproto_deps, 5, 1, + &descriptor_table_index_5fcgo_5fmsg_2eproto_initialized, descriptor_table_protodef_index_5fcgo_5fmsg_2eproto, "index_cgo_msg.proto", 541, + &descriptor_table_index_5fcgo_5fmsg_2eproto_once, descriptor_table_index_5fcgo_5fmsg_2eproto_sccs, descriptor_table_index_5fcgo_5fmsg_2eproto_deps, 7, 1, schemas, file_default_instances, TableStruct_index_5fcgo_5fmsg_2eproto::offsets, - file_level_metadata_index_5fcgo_5fmsg_2eproto, 5, file_level_enum_descriptors_index_5fcgo_5fmsg_2eproto, file_level_service_descriptors_index_5fcgo_5fmsg_2eproto, + file_level_metadata_index_5fcgo_5fmsg_2eproto, 7, file_level_enum_descriptors_index_5fcgo_5fmsg_2eproto, file_level_service_descriptors_index_5fcgo_5fmsg_2eproto, }; // Force running AddDescriptors() at dynamic initialization time. @@ -1000,6 +1061,370 @@ void MapParams::InternalSwap(MapParams* other) { } +// =================================================================== + +MapParamsV2_ParamsEntry_DoNotUse::MapParamsV2_ParamsEntry_DoNotUse() {} +MapParamsV2_ParamsEntry_DoNotUse::MapParamsV2_ParamsEntry_DoNotUse(::PROTOBUF_NAMESPACE_ID::Arena* arena) + : SuperType(arena) {} +void MapParamsV2_ParamsEntry_DoNotUse::MergeFrom(const MapParamsV2_ParamsEntry_DoNotUse& other) { + MergeFromInternal(other); +} +::PROTOBUF_NAMESPACE_ID::Metadata MapParamsV2_ParamsEntry_DoNotUse::GetMetadata() const { + return GetMetadataStatic(); +} +void MapParamsV2_ParamsEntry_DoNotUse::MergeFrom( + const ::PROTOBUF_NAMESPACE_ID::Message& other) { + ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom(other); +} + + +// =================================================================== + +void MapParamsV2::InitAsDefaultInstance() { +} +class MapParamsV2::_Internal { + public: +}; + +MapParamsV2::MapParamsV2() + : ::PROTOBUF_NAMESPACE_ID::Message(), _internal_metadata_(nullptr) { + SharedCtor(); + // @@protoc_insertion_point(constructor:milvus.proto.indexcgo.MapParamsV2) +} +MapParamsV2::MapParamsV2(const MapParamsV2& from) + : ::PROTOBUF_NAMESPACE_ID::Message(), + _internal_metadata_(nullptr) { + _internal_metadata_.MergeFrom(from._internal_metadata_); + params_.MergeFrom(from.params_); + // @@protoc_insertion_point(copy_constructor:milvus.proto.indexcgo.MapParamsV2) +} + +void MapParamsV2::SharedCtor() { + ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_MapParamsV2_index_5fcgo_5fmsg_2eproto.base); +} + +MapParamsV2::~MapParamsV2() { + // @@protoc_insertion_point(destructor:milvus.proto.indexcgo.MapParamsV2) + SharedDtor(); +} + +void MapParamsV2::SharedDtor() { +} + +void MapParamsV2::SetCachedSize(int size) const { + _cached_size_.Set(size); +} +const MapParamsV2& MapParamsV2::default_instance() { + ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&::scc_info_MapParamsV2_index_5fcgo_5fmsg_2eproto.base); + return *internal_default_instance(); +} + + +void MapParamsV2::Clear() { +// @@protoc_insertion_point(message_clear_start:milvus.proto.indexcgo.MapParamsV2) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + // Prevent compiler warnings about cached_has_bits being unused + (void) cached_has_bits; + + params_.Clear(); + _internal_metadata_.Clear(); +} + +#if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER +const char* MapParamsV2::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) { +#define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure + while (!ctx->Done(&ptr)) { + ::PROTOBUF_NAMESPACE_ID::uint32 tag; + ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag); + CHK_(ptr); + switch (tag >> 3) { + // map params = 1; + case 1: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 10)) { + ptr -= 1; + do { + ptr += 1; + ptr = ctx->ParseMessage(¶ms_, ptr); + CHK_(ptr); + if (!ctx->DataAvailable(ptr)) break; + } while (::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<::PROTOBUF_NAMESPACE_ID::uint8>(ptr) == 10); + } else goto handle_unusual; + continue; + default: { + handle_unusual: + if ((tag & 7) == 4 || tag == 0) { + ctx->SetLastTag(tag); + goto success; + } + ptr = UnknownFieldParse(tag, &_internal_metadata_, ptr, ctx); + CHK_(ptr != nullptr); + continue; + } + } // switch + } // while +success: + return ptr; +failure: + ptr = nullptr; + goto success; +#undef CHK_ +} +#else // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER +bool MapParamsV2::MergePartialFromCodedStream( + ::PROTOBUF_NAMESPACE_ID::io::CodedInputStream* input) { +#define DO_(EXPRESSION) if (!PROTOBUF_PREDICT_TRUE(EXPRESSION)) goto failure + ::PROTOBUF_NAMESPACE_ID::uint32 tag; + // @@protoc_insertion_point(parse_start:milvus.proto.indexcgo.MapParamsV2) + for (;;) { + ::std::pair<::PROTOBUF_NAMESPACE_ID::uint32, bool> p = input->ReadTagWithCutoffNoLastTag(127u); + tag = p.first; + if (!p.second) goto handle_unusual; + switch (::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::GetTagFieldNumber(tag)) { + // map params = 1; + case 1: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (10 & 0xFF)) { + MapParamsV2_ParamsEntry_DoNotUse::Parser< ::PROTOBUF_NAMESPACE_ID::internal::MapField< + MapParamsV2_ParamsEntry_DoNotUse, + std::string, std::string, + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_STRING, + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_STRING, + 0 >, + ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string > > parser(¶ms_); + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadMessageNoVirtual( + input, &parser)); + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + parser.key().data(), static_cast(parser.key().length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, + "milvus.proto.indexcgo.MapParamsV2.ParamsEntry.key")); + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + parser.value().data(), static_cast(parser.value().length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, + "milvus.proto.indexcgo.MapParamsV2.ParamsEntry.value")); + } else { + goto handle_unusual; + } + break; + } + + default: { + handle_unusual: + if (tag == 0) { + goto success; + } + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SkipField( + input, tag, _internal_metadata_.mutable_unknown_fields())); + break; + } + } + } +success: + // @@protoc_insertion_point(parse_success:milvus.proto.indexcgo.MapParamsV2) + return true; +failure: + // @@protoc_insertion_point(parse_failure:milvus.proto.indexcgo.MapParamsV2) + return false; +#undef DO_ +} +#endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER + +void MapParamsV2::SerializeWithCachedSizes( + ::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream* output) const { + // @@protoc_insertion_point(serialize_start:milvus.proto.indexcgo.MapParamsV2) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + (void) cached_has_bits; + + // map params = 1; + if (!this->params().empty()) { + typedef ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >::const_pointer + ConstPtr; + typedef ConstPtr SortItem; + typedef ::PROTOBUF_NAMESPACE_ID::internal::CompareByDerefFirst Less; + struct Utf8Check { + static void Check(ConstPtr p) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + p->first.data(), static_cast(p->first.length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, + "milvus.proto.indexcgo.MapParamsV2.ParamsEntry.key"); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + p->second.data(), static_cast(p->second.length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, + "milvus.proto.indexcgo.MapParamsV2.ParamsEntry.value"); + } + }; + + if (output->IsSerializationDeterministic() && + this->params().size() > 1) { + ::std::unique_ptr items( + new SortItem[this->params().size()]); + typedef ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >::size_type size_type; + size_type n = 0; + for (::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >::const_iterator + it = this->params().begin(); + it != this->params().end(); ++it, ++n) { + items[static_cast(n)] = SortItem(&*it); + } + ::std::sort(&items[0], &items[static_cast(n)], Less()); + for (size_type i = 0; i < n; i++) { + MapParamsV2_ParamsEntry_DoNotUse::Funcs::SerializeToCodedStream(1, items[static_cast(i)]->first, items[static_cast(i)]->second, output); + Utf8Check::Check(&(*items[static_cast(i)])); + } + } else { + for (::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >::const_iterator + it = this->params().begin(); + it != this->params().end(); ++it) { + MapParamsV2_ParamsEntry_DoNotUse::Funcs::SerializeToCodedStream(1, it->first, it->second, output); + Utf8Check::Check(&(*it)); + } + } + } + + if (_internal_metadata_.have_unknown_fields()) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFields( + _internal_metadata_.unknown_fields(), output); + } + // @@protoc_insertion_point(serialize_end:milvus.proto.indexcgo.MapParamsV2) +} + +::PROTOBUF_NAMESPACE_ID::uint8* MapParamsV2::InternalSerializeWithCachedSizesToArray( + ::PROTOBUF_NAMESPACE_ID::uint8* target) const { + // @@protoc_insertion_point(serialize_to_array_start:milvus.proto.indexcgo.MapParamsV2) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + (void) cached_has_bits; + + // map params = 1; + if (!this->params().empty()) { + typedef ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >::const_pointer + ConstPtr; + typedef ConstPtr SortItem; + typedef ::PROTOBUF_NAMESPACE_ID::internal::CompareByDerefFirst Less; + struct Utf8Check { + static void Check(ConstPtr p) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + p->first.data(), static_cast(p->first.length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, + "milvus.proto.indexcgo.MapParamsV2.ParamsEntry.key"); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + p->second.data(), static_cast(p->second.length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, + "milvus.proto.indexcgo.MapParamsV2.ParamsEntry.value"); + } + }; + + if (false && + this->params().size() > 1) { + ::std::unique_ptr items( + new SortItem[this->params().size()]); + typedef ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >::size_type size_type; + size_type n = 0; + for (::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >::const_iterator + it = this->params().begin(); + it != this->params().end(); ++it, ++n) { + items[static_cast(n)] = SortItem(&*it); + } + ::std::sort(&items[0], &items[static_cast(n)], Less()); + for (size_type i = 0; i < n; i++) { + target = MapParamsV2_ParamsEntry_DoNotUse::Funcs::SerializeToArray(1, items[static_cast(i)]->first, items[static_cast(i)]->second, target); + Utf8Check::Check(&(*items[static_cast(i)])); + } + } else { + for (::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >::const_iterator + it = this->params().begin(); + it != this->params().end(); ++it) { + target = MapParamsV2_ParamsEntry_DoNotUse::Funcs::SerializeToArray(1, it->first, it->second, target); + Utf8Check::Check(&(*it)); + } + } + } + + if (_internal_metadata_.have_unknown_fields()) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFieldsToArray( + _internal_metadata_.unknown_fields(), target); + } + // @@protoc_insertion_point(serialize_to_array_end:milvus.proto.indexcgo.MapParamsV2) + return target; +} + +size_t MapParamsV2::ByteSizeLong() const { +// @@protoc_insertion_point(message_byte_size_start:milvus.proto.indexcgo.MapParamsV2) + size_t total_size = 0; + + if (_internal_metadata_.have_unknown_fields()) { + total_size += + ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::ComputeUnknownFieldsSize( + _internal_metadata_.unknown_fields()); + } + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + // Prevent compiler warnings about cached_has_bits being unused + (void) cached_has_bits; + + // map params = 1; + total_size += 1 * + ::PROTOBUF_NAMESPACE_ID::internal::FromIntSize(this->params_size()); + for (::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >::const_iterator + it = this->params().begin(); + it != this->params().end(); ++it) { + total_size += MapParamsV2_ParamsEntry_DoNotUse::Funcs::ByteSizeLong(it->first, it->second); + } + + int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size); + SetCachedSize(cached_size); + return total_size; +} + +void MapParamsV2::MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { +// @@protoc_insertion_point(generalized_merge_from_start:milvus.proto.indexcgo.MapParamsV2) + GOOGLE_DCHECK_NE(&from, this); + const MapParamsV2* source = + ::PROTOBUF_NAMESPACE_ID::DynamicCastToGenerated( + &from); + if (source == nullptr) { + // @@protoc_insertion_point(generalized_merge_from_cast_fail:milvus.proto.indexcgo.MapParamsV2) + ::PROTOBUF_NAMESPACE_ID::internal::ReflectionOps::Merge(from, this); + } else { + // @@protoc_insertion_point(generalized_merge_from_cast_success:milvus.proto.indexcgo.MapParamsV2) + MergeFrom(*source); + } +} + +void MapParamsV2::MergeFrom(const MapParamsV2& from) { +// @@protoc_insertion_point(class_specific_merge_from_start:milvus.proto.indexcgo.MapParamsV2) + GOOGLE_DCHECK_NE(&from, this); + _internal_metadata_.MergeFrom(from._internal_metadata_); + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + (void) cached_has_bits; + + params_.MergeFrom(from.params_); +} + +void MapParamsV2::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { +// @@protoc_insertion_point(generalized_copy_from_start:milvus.proto.indexcgo.MapParamsV2) + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +void MapParamsV2::CopyFrom(const MapParamsV2& from) { +// @@protoc_insertion_point(class_specific_copy_from_start:milvus.proto.indexcgo.MapParamsV2) + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool MapParamsV2::IsInitialized() const { + return true; +} + +void MapParamsV2::InternalSwap(MapParamsV2* other) { + using std::swap; + _internal_metadata_.Swap(&other->_internal_metadata_); + params_.Swap(&other->params_); +} + +::PROTOBUF_NAMESPACE_ID::Metadata MapParamsV2::GetMetadata() const { + return GetMetadataStatic(); +} + + // =================================================================== void Binary::InitAsDefaultInstance() { @@ -1595,6 +2020,12 @@ template<> PROTOBUF_NOINLINE ::milvus::proto::indexcgo::IndexParams* Arena::Crea template<> PROTOBUF_NOINLINE ::milvus::proto::indexcgo::MapParams* Arena::CreateMaybeMessage< ::milvus::proto::indexcgo::MapParams >(Arena* arena) { return Arena::CreateInternal< ::milvus::proto::indexcgo::MapParams >(arena); } +template<> PROTOBUF_NOINLINE ::milvus::proto::indexcgo::MapParamsV2_ParamsEntry_DoNotUse* Arena::CreateMaybeMessage< ::milvus::proto::indexcgo::MapParamsV2_ParamsEntry_DoNotUse >(Arena* arena) { + return Arena::CreateInternal< ::milvus::proto::indexcgo::MapParamsV2_ParamsEntry_DoNotUse >(arena); +} +template<> PROTOBUF_NOINLINE ::milvus::proto::indexcgo::MapParamsV2* Arena::CreateMaybeMessage< ::milvus::proto::indexcgo::MapParamsV2 >(Arena* arena) { + return Arena::CreateInternal< ::milvus::proto::indexcgo::MapParamsV2 >(arena); +} template<> PROTOBUF_NOINLINE ::milvus::proto::indexcgo::Binary* Arena::CreateMaybeMessage< ::milvus::proto::indexcgo::Binary >(Arena* arena) { return Arena::CreateInternal< ::milvus::proto::indexcgo::Binary >(arena); } diff --git a/internal/core/src/pb/index_cgo_msg.pb.h b/internal/core/src/pb/index_cgo_msg.pb.h index ea6535683c8ae0a107dbc006747911d10c5f5f5a..406bb03fefcca900eb4e869b5a90dd8252767934 100644 --- a/internal/core/src/pb/index_cgo_msg.pb.h +++ b/internal/core/src/pb/index_cgo_msg.pb.h @@ -31,6 +31,9 @@ #include #include // IWYU pragma: export #include // IWYU pragma: export +#include // IWYU pragma: export +#include +#include #include #include "common.pb.h" // @@protoc_insertion_point(includes) @@ -48,7 +51,7 @@ struct TableStruct_index_5fcgo_5fmsg_2eproto { PROTOBUF_SECTION_VARIABLE(protodesc_cold); static const ::PROTOBUF_NAMESPACE_ID::internal::AuxillaryParseTableField aux[] PROTOBUF_SECTION_VARIABLE(protodesc_cold); - static const ::PROTOBUF_NAMESPACE_ID::internal::ParseTable schema[5] + static const ::PROTOBUF_NAMESPACE_ID::internal::ParseTable schema[7] PROTOBUF_SECTION_VARIABLE(protodesc_cold); static const ::PROTOBUF_NAMESPACE_ID::internal::FieldMetadata field_metadata[]; static const ::PROTOBUF_NAMESPACE_ID::internal::SerializationTable serialization_table[]; @@ -70,6 +73,12 @@ extern IndexParamsDefaultTypeInternal _IndexParams_default_instance_; class MapParams; class MapParamsDefaultTypeInternal; extern MapParamsDefaultTypeInternal _MapParams_default_instance_; +class MapParamsV2; +class MapParamsV2DefaultTypeInternal; +extern MapParamsV2DefaultTypeInternal _MapParamsV2_default_instance_; +class MapParamsV2_ParamsEntry_DoNotUse; +class MapParamsV2_ParamsEntry_DoNotUseDefaultTypeInternal; +extern MapParamsV2_ParamsEntry_DoNotUseDefaultTypeInternal _MapParamsV2_ParamsEntry_DoNotUse_default_instance_; class TypeParams; class TypeParamsDefaultTypeInternal; extern TypeParamsDefaultTypeInternal _TypeParams_default_instance_; @@ -81,6 +90,8 @@ template<> ::milvus::proto::indexcgo::Binary* Arena::CreateMaybeMessage<::milvus template<> ::milvus::proto::indexcgo::BinarySet* Arena::CreateMaybeMessage<::milvus::proto::indexcgo::BinarySet>(Arena*); template<> ::milvus::proto::indexcgo::IndexParams* Arena::CreateMaybeMessage<::milvus::proto::indexcgo::IndexParams>(Arena*); template<> ::milvus::proto::indexcgo::MapParams* Arena::CreateMaybeMessage<::milvus::proto::indexcgo::MapParams>(Arena*); +template<> ::milvus::proto::indexcgo::MapParamsV2* Arena::CreateMaybeMessage<::milvus::proto::indexcgo::MapParamsV2>(Arena*); +template<> ::milvus::proto::indexcgo::MapParamsV2_ParamsEntry_DoNotUse* Arena::CreateMaybeMessage<::milvus::proto::indexcgo::MapParamsV2_ParamsEntry_DoNotUse>(Arena*); template<> ::milvus::proto::indexcgo::TypeParams* Arena::CreateMaybeMessage<::milvus::proto::indexcgo::TypeParams>(Arena*); PROTOBUF_NAMESPACE_CLOSE namespace milvus { @@ -500,6 +511,180 @@ class MapParams : }; // ------------------------------------------------------------------- +class MapParamsV2_ParamsEntry_DoNotUse : public ::PROTOBUF_NAMESPACE_ID::internal::MapEntry { +public: + typedef ::PROTOBUF_NAMESPACE_ID::internal::MapEntry SuperType; + MapParamsV2_ParamsEntry_DoNotUse(); + MapParamsV2_ParamsEntry_DoNotUse(::PROTOBUF_NAMESPACE_ID::Arena* arena); + void MergeFrom(const MapParamsV2_ParamsEntry_DoNotUse& other); + static const MapParamsV2_ParamsEntry_DoNotUse* internal_default_instance() { return reinterpret_cast(&_MapParamsV2_ParamsEntry_DoNotUse_default_instance_); } + static bool ValidateKey(std::string* s) { + return ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String(s->data(), s->size(), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, "milvus.proto.indexcgo.MapParamsV2.ParamsEntry.key"); + } + static bool ValidateValue(std::string* s) { + return ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String(s->data(), s->size(), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, "milvus.proto.indexcgo.MapParamsV2.ParamsEntry.value"); + } + void MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& other) final; + ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; + private: + static ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadataStatic() { + ::PROTOBUF_NAMESPACE_ID::internal::AssignDescriptors(&::descriptor_table_index_5fcgo_5fmsg_2eproto); + return ::descriptor_table_index_5fcgo_5fmsg_2eproto.file_level_metadata[3]; + } + + public: +}; + +// ------------------------------------------------------------------- + +class MapParamsV2 : + public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:milvus.proto.indexcgo.MapParamsV2) */ { + public: + MapParamsV2(); + virtual ~MapParamsV2(); + + MapParamsV2(const MapParamsV2& from); + MapParamsV2(MapParamsV2&& from) noexcept + : MapParamsV2() { + *this = ::std::move(from); + } + + inline MapParamsV2& operator=(const MapParamsV2& from) { + CopyFrom(from); + return *this; + } + inline MapParamsV2& operator=(MapParamsV2&& from) noexcept { + if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) { + if (this != &from) InternalSwap(&from); + } else { + CopyFrom(from); + } + return *this; + } + + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { + return GetDescriptor(); + } + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { + return GetMetadataStatic().descriptor; + } + static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { + return GetMetadataStatic().reflection; + } + static const MapParamsV2& default_instance(); + + static void InitAsDefaultInstance(); // FOR INTERNAL USE ONLY + static inline const MapParamsV2* internal_default_instance() { + return reinterpret_cast( + &_MapParamsV2_default_instance_); + } + static constexpr int kIndexInFileMessages = + 4; + + friend void swap(MapParamsV2& a, MapParamsV2& b) { + a.Swap(&b); + } + inline void Swap(MapParamsV2* other) { + if (other == this) return; + InternalSwap(other); + } + + // implements Message ---------------------------------------------- + + inline MapParamsV2* New() const final { + return CreateMaybeMessage(nullptr); + } + + MapParamsV2* New(::PROTOBUF_NAMESPACE_ID::Arena* arena) const final { + return CreateMaybeMessage(arena); + } + void CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final; + void MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final; + void CopyFrom(const MapParamsV2& from); + void MergeFrom(const MapParamsV2& from); + PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; + bool IsInitialized() const final; + + size_t ByteSizeLong() const final; + #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER + const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; + #else + bool MergePartialFromCodedStream( + ::PROTOBUF_NAMESPACE_ID::io::CodedInputStream* input) final; + #endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER + void SerializeWithCachedSizes( + ::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream* output) const final; + ::PROTOBUF_NAMESPACE_ID::uint8* InternalSerializeWithCachedSizesToArray( + ::PROTOBUF_NAMESPACE_ID::uint8* target) const final; + int GetCachedSize() const final { return _cached_size_.Get(); } + + private: + inline void SharedCtor(); + inline void SharedDtor(); + void SetCachedSize(int size) const final; + void InternalSwap(MapParamsV2* other); + friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; + static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { + return "milvus.proto.indexcgo.MapParamsV2"; + } + private: + inline ::PROTOBUF_NAMESPACE_ID::Arena* GetArenaNoVirtual() const { + return nullptr; + } + inline void* MaybeArenaPtr() const { + return nullptr; + } + public: + + ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; + private: + static ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadataStatic() { + ::PROTOBUF_NAMESPACE_ID::internal::AssignDescriptors(&::descriptor_table_index_5fcgo_5fmsg_2eproto); + return ::descriptor_table_index_5fcgo_5fmsg_2eproto.file_level_metadata[kIndexInFileMessages]; + } + + public: + + // nested types ---------------------------------------------------- + + + // accessors ------------------------------------------------------- + + enum : int { + kParamsFieldNumber = 1, + }; + // map params = 1; + int params_size() const; + void clear_params(); + const ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >& + params() const; + ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >* + mutable_params(); + + // @@protoc_insertion_point(class_scope:milvus.proto.indexcgo.MapParamsV2) + private: + class _Internal; + + ::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_; + ::PROTOBUF_NAMESPACE_ID::internal::MapField< + MapParamsV2_ParamsEntry_DoNotUse, + std::string, std::string, + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_STRING, + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_STRING, + 0 > params_; + mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; + friend struct ::TableStruct_index_5fcgo_5fmsg_2eproto; +}; +// ------------------------------------------------------------------- + class Binary : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:milvus.proto.indexcgo.Binary) */ { public: @@ -542,7 +727,7 @@ class Binary : &_Binary_default_instance_); } static constexpr int kIndexInFileMessages = - 3; + 5; friend void swap(Binary& a, Binary& b) { a.Swap(&b); @@ -692,7 +877,7 @@ class BinarySet : &_BinarySet_default_instance_); } static constexpr int kIndexInFileMessages = - 4; + 6; friend void swap(BinarySet& a, BinarySet& b) { a.Swap(&b); @@ -887,6 +1072,30 @@ MapParams::params() const { // ------------------------------------------------------------------- +// ------------------------------------------------------------------- + +// MapParamsV2 + +// map params = 1; +inline int MapParamsV2::params_size() const { + return params_.size(); +} +inline void MapParamsV2::clear_params() { + params_.Clear(); +} +inline const ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >& +MapParamsV2::params() const { + // @@protoc_insertion_point(field_map:milvus.proto.indexcgo.MapParamsV2.params) + return params_.GetMap(); +} +inline ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >* +MapParamsV2::mutable_params() { + // @@protoc_insertion_point(field_mutable_map:milvus.proto.indexcgo.MapParamsV2.params) + return params_.MutableMap(); +} + +// ------------------------------------------------------------------- + // Binary // string key = 1; @@ -1036,6 +1245,10 @@ BinarySet::datas() const { // ------------------------------------------------------------------- +// ------------------------------------------------------------------- + +// ------------------------------------------------------------------- + // @@protoc_insertion_point(namespace_scope) diff --git a/internal/core/src/pb/milvus.pb.cc b/internal/core/src/pb/milvus.pb.cc index 408e43e7ff31f069694a8f4e1b1f013bb564d745..1138155fa851a2a3a2549f7fec277d21ca2764f4 100644 --- a/internal/core/src/pb/milvus.pb.cc +++ b/internal/core/src/pb/milvus.pb.cc @@ -1871,6 +1871,7 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT PROTOBUF_FIELD_OFFSET(::milvus::proto::milvus::CreateIndexRequest, collection_name_), PROTOBUF_FIELD_OFFSET(::milvus::proto::milvus::CreateIndexRequest, field_name_), PROTOBUF_FIELD_OFFSET(::milvus::proto::milvus::CreateIndexRequest, extra_params_), + PROTOBUF_FIELD_OFFSET(::milvus::proto::milvus::CreateIndexRequest, index_name_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::milvus::proto::milvus::DescribeIndexRequest, _internal_metadata_), ~0u, // no _extensions_ @@ -2338,57 +2339,57 @@ static const ::PROTOBUF_NAMESPACE_ID::internal::MigrationSchema schemas[] PROTOB { 248, -1, sizeof(::milvus::proto::milvus::ShowSegmentsRequest)}, { 256, -1, sizeof(::milvus::proto::milvus::ShowSegmentsResponse)}, { 263, -1, sizeof(::milvus::proto::milvus::CreateIndexRequest)}, - { 273, -1, sizeof(::milvus::proto::milvus::DescribeIndexRequest)}, - { 283, -1, sizeof(::milvus::proto::milvus::IndexDescription)}, - { 292, -1, sizeof(::milvus::proto::milvus::DescribeIndexResponse)}, - { 299, -1, sizeof(::milvus::proto::milvus::GetIndexBuildProgressRequest)}, - { 309, -1, sizeof(::milvus::proto::milvus::GetIndexBuildProgressResponse)}, - { 317, -1, sizeof(::milvus::proto::milvus::GetIndexStateRequest)}, - { 327, -1, sizeof(::milvus::proto::milvus::GetIndexStateResponse)}, - { 335, -1, sizeof(::milvus::proto::milvus::DropIndexRequest)}, - { 345, -1, sizeof(::milvus::proto::milvus::InsertRequest)}, - { 357, -1, sizeof(::milvus::proto::milvus::MutationResult)}, - { 371, -1, sizeof(::milvus::proto::milvus::DeleteRequest)}, - { 382, -1, sizeof(::milvus::proto::milvus::PlaceholderValue)}, - { 390, -1, sizeof(::milvus::proto::milvus::PlaceholderGroup)}, - { 396, -1, sizeof(::milvus::proto::milvus::SearchRequest)}, - { 412, -1, sizeof(::milvus::proto::milvus::Hits)}, - { 420, -1, sizeof(::milvus::proto::milvus::SearchResults)}, - { 428, -1, sizeof(::milvus::proto::milvus::FlushRequest)}, - { 436, 443, sizeof(::milvus::proto::milvus::FlushResponse_CollSegIDsEntry_DoNotUse)}, - { 445, -1, sizeof(::milvus::proto::milvus::FlushResponse)}, - { 453, -1, sizeof(::milvus::proto::milvus::QueryRequest)}, - { 466, -1, sizeof(::milvus::proto::milvus::QueryResults)}, - { 474, -1, sizeof(::milvus::proto::milvus::VectorIDs)}, - { 483, -1, sizeof(::milvus::proto::milvus::VectorsArray)}, - { 491, -1, sizeof(::milvus::proto::milvus::CalcDistanceRequest)}, - { 500, -1, sizeof(::milvus::proto::milvus::CalcDistanceResults)}, - { 509, -1, sizeof(::milvus::proto::milvus::PersistentSegmentInfo)}, - { 519, -1, sizeof(::milvus::proto::milvus::GetPersistentSegmentInfoRequest)}, - { 527, -1, sizeof(::milvus::proto::milvus::GetPersistentSegmentInfoResponse)}, - { 534, -1, sizeof(::milvus::proto::milvus::QuerySegmentInfo)}, - { 548, -1, sizeof(::milvus::proto::milvus::GetQuerySegmentInfoRequest)}, - { 556, -1, sizeof(::milvus::proto::milvus::GetQuerySegmentInfoResponse)}, - { 563, -1, sizeof(::milvus::proto::milvus::DummyRequest)}, - { 569, -1, sizeof(::milvus::proto::milvus::DummyResponse)}, - { 575, -1, sizeof(::milvus::proto::milvus::RegisterLinkRequest)}, - { 580, -1, sizeof(::milvus::proto::milvus::RegisterLinkResponse)}, - { 587, -1, sizeof(::milvus::proto::milvus::GetMetricsRequest)}, - { 594, -1, sizeof(::milvus::proto::milvus::GetMetricsResponse)}, - { 602, -1, sizeof(::milvus::proto::milvus::LoadBalanceRequest)}, - { 611, -1, sizeof(::milvus::proto::milvus::ManualCompactionRequest)}, - { 618, -1, sizeof(::milvus::proto::milvus::ManualCompactionResponse)}, - { 625, -1, sizeof(::milvus::proto::milvus::GetCompactionStateRequest)}, - { 631, -1, sizeof(::milvus::proto::milvus::GetCompactionStateResponse)}, - { 641, -1, sizeof(::milvus::proto::milvus::GetCompactionPlansRequest)}, - { 647, -1, sizeof(::milvus::proto::milvus::GetCompactionPlansResponse)}, - { 655, -1, sizeof(::milvus::proto::milvus::CompactionMergeInfo)}, - { 662, -1, sizeof(::milvus::proto::milvus::GetFlushStateRequest)}, - { 668, -1, sizeof(::milvus::proto::milvus::GetFlushStateResponse)}, - { 675, -1, sizeof(::milvus::proto::milvus::ImportRequest)}, - { 685, -1, sizeof(::milvus::proto::milvus::ImportResponse)}, - { 692, -1, sizeof(::milvus::proto::milvus::GetImportStateRequest)}, - { 698, -1, sizeof(::milvus::proto::milvus::GetImportStateResponse)}, + { 274, -1, sizeof(::milvus::proto::milvus::DescribeIndexRequest)}, + { 284, -1, sizeof(::milvus::proto::milvus::IndexDescription)}, + { 293, -1, sizeof(::milvus::proto::milvus::DescribeIndexResponse)}, + { 300, -1, sizeof(::milvus::proto::milvus::GetIndexBuildProgressRequest)}, + { 310, -1, sizeof(::milvus::proto::milvus::GetIndexBuildProgressResponse)}, + { 318, -1, sizeof(::milvus::proto::milvus::GetIndexStateRequest)}, + { 328, -1, sizeof(::milvus::proto::milvus::GetIndexStateResponse)}, + { 336, -1, sizeof(::milvus::proto::milvus::DropIndexRequest)}, + { 346, -1, sizeof(::milvus::proto::milvus::InsertRequest)}, + { 358, -1, sizeof(::milvus::proto::milvus::MutationResult)}, + { 372, -1, sizeof(::milvus::proto::milvus::DeleteRequest)}, + { 383, -1, sizeof(::milvus::proto::milvus::PlaceholderValue)}, + { 391, -1, sizeof(::milvus::proto::milvus::PlaceholderGroup)}, + { 397, -1, sizeof(::milvus::proto::milvus::SearchRequest)}, + { 413, -1, sizeof(::milvus::proto::milvus::Hits)}, + { 421, -1, sizeof(::milvus::proto::milvus::SearchResults)}, + { 429, -1, sizeof(::milvus::proto::milvus::FlushRequest)}, + { 437, 444, sizeof(::milvus::proto::milvus::FlushResponse_CollSegIDsEntry_DoNotUse)}, + { 446, -1, sizeof(::milvus::proto::milvus::FlushResponse)}, + { 454, -1, sizeof(::milvus::proto::milvus::QueryRequest)}, + { 467, -1, sizeof(::milvus::proto::milvus::QueryResults)}, + { 475, -1, sizeof(::milvus::proto::milvus::VectorIDs)}, + { 484, -1, sizeof(::milvus::proto::milvus::VectorsArray)}, + { 492, -1, sizeof(::milvus::proto::milvus::CalcDistanceRequest)}, + { 501, -1, sizeof(::milvus::proto::milvus::CalcDistanceResults)}, + { 510, -1, sizeof(::milvus::proto::milvus::PersistentSegmentInfo)}, + { 520, -1, sizeof(::milvus::proto::milvus::GetPersistentSegmentInfoRequest)}, + { 528, -1, sizeof(::milvus::proto::milvus::GetPersistentSegmentInfoResponse)}, + { 535, -1, sizeof(::milvus::proto::milvus::QuerySegmentInfo)}, + { 549, -1, sizeof(::milvus::proto::milvus::GetQuerySegmentInfoRequest)}, + { 557, -1, sizeof(::milvus::proto::milvus::GetQuerySegmentInfoResponse)}, + { 564, -1, sizeof(::milvus::proto::milvus::DummyRequest)}, + { 570, -1, sizeof(::milvus::proto::milvus::DummyResponse)}, + { 576, -1, sizeof(::milvus::proto::milvus::RegisterLinkRequest)}, + { 581, -1, sizeof(::milvus::proto::milvus::RegisterLinkResponse)}, + { 588, -1, sizeof(::milvus::proto::milvus::GetMetricsRequest)}, + { 595, -1, sizeof(::milvus::proto::milvus::GetMetricsResponse)}, + { 603, -1, sizeof(::milvus::proto::milvus::LoadBalanceRequest)}, + { 612, -1, sizeof(::milvus::proto::milvus::ManualCompactionRequest)}, + { 619, -1, sizeof(::milvus::proto::milvus::ManualCompactionResponse)}, + { 626, -1, sizeof(::milvus::proto::milvus::GetCompactionStateRequest)}, + { 632, -1, sizeof(::milvus::proto::milvus::GetCompactionStateResponse)}, + { 642, -1, sizeof(::milvus::proto::milvus::GetCompactionPlansRequest)}, + { 648, -1, sizeof(::milvus::proto::milvus::GetCompactionPlansResponse)}, + { 656, -1, sizeof(::milvus::proto::milvus::CompactionMergeInfo)}, + { 663, -1, sizeof(::milvus::proto::milvus::GetFlushStateRequest)}, + { 669, -1, sizeof(::milvus::proto::milvus::GetFlushStateResponse)}, + { 676, -1, sizeof(::milvus::proto::milvus::ImportRequest)}, + { 686, -1, sizeof(::milvus::proto::milvus::ImportResponse)}, + { 693, -1, sizeof(::milvus::proto::milvus::GetImportStateRequest)}, + { 699, -1, sizeof(::milvus::proto::milvus::GetImportStateResponse)}, }; static ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] = { @@ -2584,288 +2585,288 @@ const char descriptor_table_protodef_milvus_2eproto[] PROTOBUF_SECTION_VARIABLE( "collectionID\030\002 \001(\003\022\023\n\013partitionID\030\003 \001(\003\"" "W\n\024ShowSegmentsResponse\022+\n\006status\030\001 \001(\0132" "\033.milvus.proto.common.Status\022\022\n\nsegmentI" - "Ds\030\002 \003(\003\"\267\001\n\022CreateIndexRequest\022*\n\004base\030" + "Ds\030\002 \003(\003\"\313\001\n\022CreateIndexRequest\022*\n\004base\030" "\001 \001(\0132\034.milvus.proto.common.MsgBase\022\017\n\007d" "b_name\030\002 \001(\t\022\027\n\017collection_name\030\003 \001(\t\022\022\n" "\nfield_name\030\004 \001(\t\0227\n\014extra_params\030\005 \003(\0132" - "!.milvus.proto.common.KeyValuePair\"\224\001\n\024D" - "escribeIndexRequest\022*\n\004base\030\001 \001(\0132\034.milv" - "us.proto.common.MsgBase\022\017\n\007db_name\030\002 \001(\t" - "\022\027\n\017collection_name\030\003 \001(\t\022\022\n\nfield_name\030" - "\004 \001(\t\022\022\n\nindex_name\030\005 \001(\t\"~\n\020IndexDescri" - "ption\022\022\n\nindex_name\030\001 \001(\t\022\017\n\007indexID\030\002 \001" - "(\003\0221\n\006params\030\003 \003(\0132!.milvus.proto.common" - ".KeyValuePair\022\022\n\nfield_name\030\004 \001(\t\"\207\001\n\025De" - "scribeIndexResponse\022+\n\006status\030\001 \001(\0132\033.mi" - "lvus.proto.common.Status\022A\n\022index_descri" - "ptions\030\002 \003(\0132%.milvus.proto.milvus.Index" - "Description\"\234\001\n\034GetIndexBuildProgressReq" - "uest\022*\n\004base\030\001 \001(\0132\034.milvus.proto.common" - ".MsgBase\022\017\n\007db_name\030\002 \001(\t\022\027\n\017collection_" - "name\030\003 \001(\t\022\022\n\nfield_name\030\004 \001(\t\022\022\n\nindex_" - "name\030\005 \001(\t\"v\n\035GetIndexBuildProgressRespo" - "nse\022+\n\006status\030\001 \001(\0132\033.milvus.proto.commo" - "n.Status\022\024\n\014indexed_rows\030\002 \001(\003\022\022\n\ntotal_" - "rows\030\003 \001(\003\"\224\001\n\024GetIndexStateRequest\022*\n\004b" - "ase\030\001 \001(\0132\034.milvus.proto.common.MsgBase\022" - "\017\n\007db_name\030\002 \001(\t\022\027\n\017collection_name\030\003 \001(" - "\t\022\022\n\nfield_name\030\004 \001(\t\022\022\n\nindex_name\030\005 \001(" - "\t\"\211\001\n\025GetIndexStateResponse\022+\n\006status\030\001 " - "\001(\0132\033.milvus.proto.common.Status\022.\n\005stat" - "e\030\002 \001(\0162\037.milvus.proto.common.IndexState" - "\022\023\n\013fail_reason\030\003 \001(\t\"\220\001\n\020DropIndexReque" - "st\022*\n\004base\030\001 \001(\0132\034.milvus.proto.common.M" - "sgBase\022\017\n\007db_name\030\002 \001(\t\022\027\n\017collection_na" - "me\030\003 \001(\t\022\022\n\nfield_name\030\004 \001(\t\022\022\n\nindex_na" - "me\030\005 \001(\t\"\327\001\n\rInsertRequest\022*\n\004base\030\001 \001(\013" - "2\034.milvus.proto.common.MsgBase\022\017\n\007db_nam" - "e\030\002 \001(\t\022\027\n\017collection_name\030\003 \001(\t\022\026\n\016part" - "ition_name\030\004 \001(\t\0223\n\013fields_data\030\005 \003(\0132\036." - "milvus.proto.schema.FieldData\022\021\n\thash_ke" - "ys\030\006 \003(\r\022\020\n\010num_rows\030\007 \001(\r\"\360\001\n\016MutationR" - "esult\022+\n\006status\030\001 \001(\0132\033.milvus.proto.com" - "mon.Status\022%\n\003IDs\030\002 \001(\0132\030.milvus.proto.s" - "chema.IDs\022\022\n\nsucc_index\030\003 \003(\r\022\021\n\terr_ind" - "ex\030\004 \003(\r\022\024\n\014acknowledged\030\005 \001(\010\022\022\n\ninsert" - "_cnt\030\006 \001(\003\022\022\n\ndelete_cnt\030\007 \001(\003\022\022\n\nupsert" - "_cnt\030\010 \001(\003\022\021\n\ttimestamp\030\t \001(\004\"\236\001\n\rDelete" - "Request\022*\n\004base\030\001 \001(\0132\034.milvus.proto.com" - "mon.MsgBase\022\017\n\007db_name\030\002 \001(\t\022\027\n\017collecti" - "on_name\030\003 \001(\t\022\026\n\016partition_name\030\004 \001(\t\022\014\n" - "\004expr\030\005 \001(\t\022\021\n\thash_keys\030\006 \003(\r\"c\n\020Placeh" - "olderValue\022\013\n\003tag\030\001 \001(\t\0222\n\004type\030\002 \001(\0162$." - "milvus.proto.milvus.PlaceholderType\022\016\n\006v" - "alues\030\003 \003(\014\"O\n\020PlaceholderGroup\022;\n\014place" - "holders\030\001 \003(\0132%.milvus.proto.milvus.Plac" - "eholderValue\"\336\002\n\rSearchRequest\022*\n\004base\030\001" - " \001(\0132\034.milvus.proto.common.MsgBase\022\017\n\007db" - "_name\030\002 \001(\t\022\027\n\017collection_name\030\003 \001(\t\022\027\n\017" - "partition_names\030\004 \003(\t\022\013\n\003dsl\030\005 \001(\t\022\031\n\021pl" - "aceholder_group\030\006 \001(\014\022.\n\010dsl_type\030\007 \001(\0162" - "\034.milvus.proto.common.DslType\022\025\n\routput_" - "fields\030\010 \003(\t\0228\n\rsearch_params\030\t \003(\0132!.mi" - "lvus.proto.common.KeyValuePair\022\030\n\020travel" - "_timestamp\030\n \001(\004\022\033\n\023guarantee_timestamp\030" - "\013 \001(\004\"5\n\004Hits\022\013\n\003IDs\030\001 \003(\003\022\020\n\010row_data\030\002" - " \003(\014\022\016\n\006scores\030\003 \003(\002\"\215\001\n\rSearchResults\022+" - "\n\006status\030\001 \001(\0132\033.milvus.proto.common.Sta" - "tus\0226\n\007results\030\002 \001(\0132%.milvus.proto.sche" - "ma.SearchResultData\022\027\n\017collection_name\030\003" - " \001(\t\"e\n\014FlushRequest\022*\n\004base\030\001 \001(\0132\034.mil" - "vus.proto.common.MsgBase\022\017\n\007db_name\030\002 \001(" - "\t\022\030\n\020collection_names\030\003 \003(\t\"\351\001\n\rFlushRes" - "ponse\022+\n\006status\030\001 \001(\0132\033.milvus.proto.com" - "mon.Status\022\017\n\007db_name\030\002 \001(\t\022G\n\013coll_segI" - "Ds\030\003 \003(\01322.milvus.proto.milvus.FlushResp" - "onse.CollSegIDsEntry\032Q\n\017CollSegIDsEntry\022" - "\013\n\003key\030\001 \001(\t\022-\n\005value\030\002 \001(\0132\036.milvus.pro" - "to.schema.LongArray:\0028\001\"\331\001\n\014QueryRequest" - "\022*\n\004base\030\001 \001(\0132\034.milvus.proto.common.Msg" - "Base\022\017\n\007db_name\030\002 \001(\t\022\027\n\017collection_name" - "\030\003 \001(\t\022\014\n\004expr\030\004 \001(\t\022\025\n\routput_fields\030\005 " - "\003(\t\022\027\n\017partition_names\030\006 \003(\t\022\030\n\020travel_t" - "imestamp\030\007 \001(\004\022\033\n\023guarantee_timestamp\030\010 " - "\001(\004\"\211\001\n\014QueryResults\022+\n\006status\030\001 \001(\0132\033.m" - "ilvus.proto.common.Status\0223\n\013fields_data" - "\030\002 \003(\0132\036.milvus.proto.schema.FieldData\022\027" - "\n\017collection_name\030\003 \001(\t\"}\n\tVectorIDs\022\027\n\017" - "collection_name\030\001 \001(\t\022\022\n\nfield_name\030\002 \001(" - "\t\022*\n\010id_array\030\003 \001(\0132\030.milvus.proto.schem" - "a.IDs\022\027\n\017partition_names\030\004 \003(\t\"\203\001\n\014Vecto" - "rsArray\0222\n\010id_array\030\001 \001(\0132\036.milvus.proto" - ".milvus.VectorIDsH\000\0226\n\ndata_array\030\002 \001(\0132" - " .milvus.proto.schema.VectorFieldH\000B\007\n\005a" - "rray\"\335\001\n\023CalcDistanceRequest\022*\n\004base\030\001 \001" - "(\0132\034.milvus.proto.common.MsgBase\0222\n\007op_l" - "eft\030\002 \001(\0132!.milvus.proto.milvus.VectorsA" - "rray\0223\n\010op_right\030\003 \001(\0132!.milvus.proto.mi" - "lvus.VectorsArray\0221\n\006params\030\004 \003(\0132!.milv" - "us.proto.common.KeyValuePair\"\265\001\n\023CalcDis" - "tanceResults\022+\n\006status\030\001 \001(\0132\033.milvus.pr" - "oto.common.Status\0221\n\010int_dist\030\002 \001(\0132\035.mi" - "lvus.proto.schema.IntArrayH\000\0225\n\nfloat_di" - "st\030\003 \001(\0132\037.milvus.proto.schema.FloatArra" - "yH\000B\007\n\005array\"\231\001\n\025PersistentSegmentInfo\022\021" - "\n\tsegmentID\030\001 \001(\003\022\024\n\014collectionID\030\002 \001(\003\022" - "\023\n\013partitionID\030\003 \001(\003\022\020\n\010num_rows\030\004 \001(\003\0220" - "\n\005state\030\005 \001(\0162!.milvus.proto.common.Segm" - "entState\"u\n\037GetPersistentSegmentInfoRequ" - "est\022*\n\004base\030\001 \001(\0132\034.milvus.proto.common." - "MsgBase\022\016\n\006dbName\030\002 \001(\t\022\026\n\016collectionNam" - "e\030\003 \001(\t\"\212\001\n GetPersistentSegmentInfoResp" - "onse\022+\n\006status\030\001 \001(\0132\033.milvus.proto.comm" - "on.Status\0229\n\005infos\030\002 \003(\0132*.milvus.proto." - "milvus.PersistentSegmentInfo\"\333\001\n\020QuerySe" - "gmentInfo\022\021\n\tsegmentID\030\001 \001(\003\022\024\n\014collecti" - "onID\030\002 \001(\003\022\023\n\013partitionID\030\003 \001(\003\022\020\n\010mem_s" - "ize\030\004 \001(\003\022\020\n\010num_rows\030\005 \001(\003\022\022\n\nindex_nam" - "e\030\006 \001(\t\022\017\n\007indexID\030\007 \001(\003\022\016\n\006nodeID\030\010 \001(\003" - "\0220\n\005state\030\t \001(\0162!.milvus.proto.common.Se" - "gmentState\"p\n\032GetQuerySegmentInfoRequest" + "!.milvus.proto.common.KeyValuePair\022\022\n\nin" + "dex_name\030\006 \001(\t\"\224\001\n\024DescribeIndexRequest\022" + "*\n\004base\030\001 \001(\0132\034.milvus.proto.common.MsgB" + "ase\022\017\n\007db_name\030\002 \001(\t\022\027\n\017collection_name\030" + "\003 \001(\t\022\022\n\nfield_name\030\004 \001(\t\022\022\n\nindex_name\030" + "\005 \001(\t\"~\n\020IndexDescription\022\022\n\nindex_name\030" + "\001 \001(\t\022\017\n\007indexID\030\002 \001(\003\0221\n\006params\030\003 \003(\0132!" + ".milvus.proto.common.KeyValuePair\022\022\n\nfie" + "ld_name\030\004 \001(\t\"\207\001\n\025DescribeIndexResponse\022" + "+\n\006status\030\001 \001(\0132\033.milvus.proto.common.St" + "atus\022A\n\022index_descriptions\030\002 \003(\0132%.milvu" + "s.proto.milvus.IndexDescription\"\234\001\n\034GetI" + "ndexBuildProgressRequest\022*\n\004base\030\001 \001(\0132\034" + ".milvus.proto.common.MsgBase\022\017\n\007db_name\030" + "\002 \001(\t\022\027\n\017collection_name\030\003 \001(\t\022\022\n\nfield_" + "name\030\004 \001(\t\022\022\n\nindex_name\030\005 \001(\t\"v\n\035GetInd" + "exBuildProgressResponse\022+\n\006status\030\001 \001(\0132" + "\033.milvus.proto.common.Status\022\024\n\014indexed_" + "rows\030\002 \001(\003\022\022\n\ntotal_rows\030\003 \001(\003\"\224\001\n\024GetIn" + "dexStateRequest\022*\n\004base\030\001 \001(\0132\034.milvus.p" + "roto.common.MsgBase\022\017\n\007db_name\030\002 \001(\t\022\027\n\017" + "collection_name\030\003 \001(\t\022\022\n\nfield_name\030\004 \001(" + "\t\022\022\n\nindex_name\030\005 \001(\t\"\211\001\n\025GetIndexStateR" + "esponse\022+\n\006status\030\001 \001(\0132\033.milvus.proto.c" + "ommon.Status\022.\n\005state\030\002 \001(\0162\037.milvus.pro" + "to.common.IndexState\022\023\n\013fail_reason\030\003 \001(" + "\t\"\220\001\n\020DropIndexRequest\022*\n\004base\030\001 \001(\0132\034.m" + "ilvus.proto.common.MsgBase\022\017\n\007db_name\030\002 " + "\001(\t\022\027\n\017collection_name\030\003 \001(\t\022\022\n\nfield_na" + "me\030\004 \001(\t\022\022\n\nindex_name\030\005 \001(\t\"\327\001\n\rInsertR" + "equest\022*\n\004base\030\001 \001(\0132\034.milvus.proto.comm" + "on.MsgBase\022\017\n\007db_name\030\002 \001(\t\022\027\n\017collectio" + "n_name\030\003 \001(\t\022\026\n\016partition_name\030\004 \001(\t\0223\n\013" + "fields_data\030\005 \003(\0132\036.milvus.proto.schema." + "FieldData\022\021\n\thash_keys\030\006 \003(\r\022\020\n\010num_rows" + "\030\007 \001(\r\"\360\001\n\016MutationResult\022+\n\006status\030\001 \001(" + "\0132\033.milvus.proto.common.Status\022%\n\003IDs\030\002 " + "\001(\0132\030.milvus.proto.schema.IDs\022\022\n\nsucc_in" + "dex\030\003 \003(\r\022\021\n\terr_index\030\004 \003(\r\022\024\n\014acknowle" + "dged\030\005 \001(\010\022\022\n\ninsert_cnt\030\006 \001(\003\022\022\n\ndelete" + "_cnt\030\007 \001(\003\022\022\n\nupsert_cnt\030\010 \001(\003\022\021\n\ttimest" + "amp\030\t \001(\004\"\236\001\n\rDeleteRequest\022*\n\004base\030\001 \001(" + "\0132\034.milvus.proto.common.MsgBase\022\017\n\007db_na" + "me\030\002 \001(\t\022\027\n\017collection_name\030\003 \001(\t\022\026\n\016par" + "tition_name\030\004 \001(\t\022\014\n\004expr\030\005 \001(\t\022\021\n\thash_" + "keys\030\006 \003(\r\"c\n\020PlaceholderValue\022\013\n\003tag\030\001 " + "\001(\t\0222\n\004type\030\002 \001(\0162$.milvus.proto.milvus." + "PlaceholderType\022\016\n\006values\030\003 \003(\014\"O\n\020Place" + "holderGroup\022;\n\014placeholders\030\001 \003(\0132%.milv" + "us.proto.milvus.PlaceholderValue\"\336\002\n\rSea" + "rchRequest\022*\n\004base\030\001 \001(\0132\034.milvus.proto." + "common.MsgBase\022\017\n\007db_name\030\002 \001(\t\022\027\n\017colle" + "ction_name\030\003 \001(\t\022\027\n\017partition_names\030\004 \003(" + "\t\022\013\n\003dsl\030\005 \001(\t\022\031\n\021placeholder_group\030\006 \001(" + "\014\022.\n\010dsl_type\030\007 \001(\0162\034.milvus.proto.commo" + "n.DslType\022\025\n\routput_fields\030\010 \003(\t\0228\n\rsear" + "ch_params\030\t \003(\0132!.milvus.proto.common.Ke" + "yValuePair\022\030\n\020travel_timestamp\030\n \001(\004\022\033\n\023" + "guarantee_timestamp\030\013 \001(\004\"5\n\004Hits\022\013\n\003IDs" + "\030\001 \003(\003\022\020\n\010row_data\030\002 \003(\014\022\016\n\006scores\030\003 \003(\002" + "\"\215\001\n\rSearchResults\022+\n\006status\030\001 \001(\0132\033.mil" + "vus.proto.common.Status\0226\n\007results\030\002 \001(\013" + "2%.milvus.proto.schema.SearchResultData\022" + "\027\n\017collection_name\030\003 \001(\t\"e\n\014FlushRequest" "\022*\n\004base\030\001 \001(\0132\034.milvus.proto.common.Msg" - "Base\022\016\n\006dbName\030\002 \001(\t\022\026\n\016collectionName\030\003" - " \001(\t\"\200\001\n\033GetQuerySegmentInfoResponse\022+\n\006" + "Base\022\017\n\007db_name\030\002 \001(\t\022\030\n\020collection_name" + "s\030\003 \003(\t\"\351\001\n\rFlushResponse\022+\n\006status\030\001 \001(" + "\0132\033.milvus.proto.common.Status\022\017\n\007db_nam" + "e\030\002 \001(\t\022G\n\013coll_segIDs\030\003 \003(\01322.milvus.pr" + "oto.milvus.FlushResponse.CollSegIDsEntry" + "\032Q\n\017CollSegIDsEntry\022\013\n\003key\030\001 \001(\t\022-\n\005valu" + "e\030\002 \001(\0132\036.milvus.proto.schema.LongArray:" + "\0028\001\"\331\001\n\014QueryRequest\022*\n\004base\030\001 \001(\0132\034.mil" + "vus.proto.common.MsgBase\022\017\n\007db_name\030\002 \001(" + "\t\022\027\n\017collection_name\030\003 \001(\t\022\014\n\004expr\030\004 \001(\t" + "\022\025\n\routput_fields\030\005 \003(\t\022\027\n\017partition_nam" + "es\030\006 \003(\t\022\030\n\020travel_timestamp\030\007 \001(\004\022\033\n\023gu" + "arantee_timestamp\030\010 \001(\004\"\211\001\n\014QueryResults" + "\022+\n\006status\030\001 \001(\0132\033.milvus.proto.common.S" + "tatus\0223\n\013fields_data\030\002 \003(\0132\036.milvus.prot" + "o.schema.FieldData\022\027\n\017collection_name\030\003 " + "\001(\t\"}\n\tVectorIDs\022\027\n\017collection_name\030\001 \001(" + "\t\022\022\n\nfield_name\030\002 \001(\t\022*\n\010id_array\030\003 \001(\0132" + "\030.milvus.proto.schema.IDs\022\027\n\017partition_n" + "ames\030\004 \003(\t\"\203\001\n\014VectorsArray\0222\n\010id_array\030" + "\001 \001(\0132\036.milvus.proto.milvus.VectorIDsH\000\022" + "6\n\ndata_array\030\002 \001(\0132 .milvus.proto.schem" + "a.VectorFieldH\000B\007\n\005array\"\335\001\n\023CalcDistanc" + "eRequest\022*\n\004base\030\001 \001(\0132\034.milvus.proto.co" + "mmon.MsgBase\0222\n\007op_left\030\002 \001(\0132!.milvus.p" + "roto.milvus.VectorsArray\0223\n\010op_right\030\003 \001" + "(\0132!.milvus.proto.milvus.VectorsArray\0221\n" + "\006params\030\004 \003(\0132!.milvus.proto.common.KeyV" + "aluePair\"\265\001\n\023CalcDistanceResults\022+\n\006stat" + "us\030\001 \001(\0132\033.milvus.proto.common.Status\0221\n" + "\010int_dist\030\002 \001(\0132\035.milvus.proto.schema.In" + "tArrayH\000\0225\n\nfloat_dist\030\003 \001(\0132\037.milvus.pr" + "oto.schema.FloatArrayH\000B\007\n\005array\"\231\001\n\025Per" + "sistentSegmentInfo\022\021\n\tsegmentID\030\001 \001(\003\022\024\n" + "\014collectionID\030\002 \001(\003\022\023\n\013partitionID\030\003 \001(\003" + "\022\020\n\010num_rows\030\004 \001(\003\0220\n\005state\030\005 \001(\0162!.milv" + "us.proto.common.SegmentState\"u\n\037GetPersi" + "stentSegmentInfoRequest\022*\n\004base\030\001 \001(\0132\034." + "milvus.proto.common.MsgBase\022\016\n\006dbName\030\002 " + "\001(\t\022\026\n\016collectionName\030\003 \001(\t\"\212\001\n GetPersi" + "stentSegmentInfoResponse\022+\n\006status\030\001 \001(\013" + "2\033.milvus.proto.common.Status\0229\n\005infos\030\002" + " \003(\0132*.milvus.proto.milvus.PersistentSeg" + "mentInfo\"\333\001\n\020QuerySegmentInfo\022\021\n\tsegment" + "ID\030\001 \001(\003\022\024\n\014collectionID\030\002 \001(\003\022\023\n\013partit" + "ionID\030\003 \001(\003\022\020\n\010mem_size\030\004 \001(\003\022\020\n\010num_row" + "s\030\005 \001(\003\022\022\n\nindex_name\030\006 \001(\t\022\017\n\007indexID\030\007" + " \001(\003\022\016\n\006nodeID\030\010 \001(\003\0220\n\005state\030\t \001(\0162!.mi" + "lvus.proto.common.SegmentState\"p\n\032GetQue" + "rySegmentInfoRequest\022*\n\004base\030\001 \001(\0132\034.mil" + "vus.proto.common.MsgBase\022\016\n\006dbName\030\002 \001(\t" + "\022\026\n\016collectionName\030\003 \001(\t\"\200\001\n\033GetQuerySeg" + "mentInfoResponse\022+\n\006status\030\001 \001(\0132\033.milvu" + "s.proto.common.Status\0224\n\005infos\030\002 \003(\0132%.m" + "ilvus.proto.milvus.QuerySegmentInfo\"$\n\014D" + "ummyRequest\022\024\n\014request_type\030\001 \001(\t\"!\n\rDum" + "myResponse\022\020\n\010response\030\001 \001(\t\"\025\n\023Register" + "LinkRequest\"r\n\024RegisterLinkResponse\022-\n\007a" + "ddress\030\001 \001(\0132\034.milvus.proto.common.Addre" + "ss\022+\n\006status\030\002 \001(\0132\033.milvus.proto.common" + ".Status\"P\n\021GetMetricsRequest\022*\n\004base\030\001 \001" + "(\0132\034.milvus.proto.common.MsgBase\022\017\n\007requ" + "est\030\002 \001(\t\"k\n\022GetMetricsResponse\022+\n\006statu" + "s\030\001 \001(\0132\033.milvus.proto.common.Status\022\020\n\010" + "response\030\002 \001(\t\022\026\n\016component_name\030\003 \001(\t\"\204" + "\001\n\022LoadBalanceRequest\022*\n\004base\030\001 \001(\0132\034.mi" + "lvus.proto.common.MsgBase\022\022\n\nsrc_nodeID\030" + "\002 \001(\003\022\023\n\013dst_nodeIDs\030\003 \003(\003\022\031\n\021sealed_seg" + "mentIDs\030\004 \003(\003\"C\n\027ManualCompactionRequest" + "\022\024\n\014collectionID\030\001 \001(\003\022\022\n\ntimetravel\030\002 \001" + "(\004\"]\n\030ManualCompactionResponse\022+\n\006status" + "\030\001 \001(\0132\033.milvus.proto.common.Status\022\024\n\014c" + "ompactionID\030\002 \001(\003\"1\n\031GetCompactionStateR" + "equest\022\024\n\014compactionID\030\001 \001(\003\"\307\001\n\032GetComp" + "actionStateResponse\022+\n\006status\030\001 \001(\0132\033.mi" + "lvus.proto.common.Status\0223\n\005state\030\002 \001(\0162" + "$.milvus.proto.common.CompactionState\022\027\n" + "\017executingPlanNo\030\003 \001(\003\022\025\n\rtimeoutPlanNo\030" + "\004 \001(\003\022\027\n\017completedPlanNo\030\005 \001(\003\"1\n\031GetCom" + "pactionPlansRequest\022\024\n\014compactionID\030\001 \001(" + "\003\"\274\001\n\032GetCompactionPlansResponse\022+\n\006stat" + "us\030\001 \001(\0132\033.milvus.proto.common.Status\0223\n" + "\005state\030\002 \001(\0162$.milvus.proto.common.Compa" + "ctionState\022<\n\nmergeInfos\030\003 \003(\0132(.milvus." + "proto.milvus.CompactionMergeInfo\"6\n\023Comp" + "actionMergeInfo\022\017\n\007sources\030\001 \003(\003\022\016\n\006targ" + "et\030\002 \001(\003\"*\n\024GetFlushStateRequest\022\022\n\nsegm" + "entIDs\030\001 \003(\003\"U\n\025GetFlushStateResponse\022+\n" + "\006status\030\001 \001(\0132\033.milvus.proto.common.Stat" + "us\022\017\n\007flushed\030\002 \001(\010\"\226\001\n\rImportRequest\022\027\n" + "\017collection_name\030\001 \001(\t\022\026\n\016partition_name" + "\030\002 \001(\t\022\021\n\trow_based\030\003 \001(\010\022\r\n\005files\030\004 \003(\t" + "\0222\n\007options\030\005 \003(\0132!.milvus.proto.common." + "KeyValuePair\"L\n\016ImportResponse\022+\n\006status" + "\030\001 \001(\0132\033.milvus.proto.common.Status\022\r\n\005t" + "asks\030\002 \003(\003\"%\n\025GetImportStateRequest\022\014\n\004t" + "ask\030\001 \001(\003\"\314\001\n\026GetImportStateResponse\022+\n\006" "status\030\001 \001(\0132\033.milvus.proto.common.Statu" - "s\0224\n\005infos\030\002 \003(\0132%.milvus.proto.milvus.Q" - "uerySegmentInfo\"$\n\014DummyRequest\022\024\n\014reque" - "st_type\030\001 \001(\t\"!\n\rDummyResponse\022\020\n\010respon" - "se\030\001 \001(\t\"\025\n\023RegisterLinkRequest\"r\n\024Regis" - "terLinkResponse\022-\n\007address\030\001 \001(\0132\034.milvu" - "s.proto.common.Address\022+\n\006status\030\002 \001(\0132\033" - ".milvus.proto.common.Status\"P\n\021GetMetric" - "sRequest\022*\n\004base\030\001 \001(\0132\034.milvus.proto.co" - "mmon.MsgBase\022\017\n\007request\030\002 \001(\t\"k\n\022GetMetr" - "icsResponse\022+\n\006status\030\001 \001(\0132\033.milvus.pro" - "to.common.Status\022\020\n\010response\030\002 \001(\t\022\026\n\016co" - "mponent_name\030\003 \001(\t\"\204\001\n\022LoadBalanceReques" - "t\022*\n\004base\030\001 \001(\0132\034.milvus.proto.common.Ms" - "gBase\022\022\n\nsrc_nodeID\030\002 \001(\003\022\023\n\013dst_nodeIDs" - "\030\003 \003(\003\022\031\n\021sealed_segmentIDs\030\004 \003(\003\"C\n\027Man" - "ualCompactionRequest\022\024\n\014collectionID\030\001 \001" - "(\003\022\022\n\ntimetravel\030\002 \001(\004\"]\n\030ManualCompacti" - "onResponse\022+\n\006status\030\001 \001(\0132\033.milvus.prot" - "o.common.Status\022\024\n\014compactionID\030\002 \001(\003\"1\n" - "\031GetCompactionStateRequest\022\024\n\014compaction" - "ID\030\001 \001(\003\"\307\001\n\032GetCompactionStateResponse\022" - "+\n\006status\030\001 \001(\0132\033.milvus.proto.common.St" - "atus\0223\n\005state\030\002 \001(\0162$.milvus.proto.commo" - "n.CompactionState\022\027\n\017executingPlanNo\030\003 \001" - "(\003\022\025\n\rtimeoutPlanNo\030\004 \001(\003\022\027\n\017completedPl" - "anNo\030\005 \001(\003\"1\n\031GetCompactionPlansRequest\022" - "\024\n\014compactionID\030\001 \001(\003\"\274\001\n\032GetCompactionP" - "lansResponse\022+\n\006status\030\001 \001(\0132\033.milvus.pr" - "oto.common.Status\0223\n\005state\030\002 \001(\0162$.milvu" - "s.proto.common.CompactionState\022<\n\nmergeI" - "nfos\030\003 \003(\0132(.milvus.proto.milvus.Compact" - "ionMergeInfo\"6\n\023CompactionMergeInfo\022\017\n\007s" - "ources\030\001 \003(\003\022\016\n\006target\030\002 \001(\003\"*\n\024GetFlush" - "StateRequest\022\022\n\nsegmentIDs\030\001 \003(\003\"U\n\025GetF" - "lushStateResponse\022+\n\006status\030\001 \001(\0132\033.milv" - "us.proto.common.Status\022\017\n\007flushed\030\002 \001(\010\"" - "\226\001\n\rImportRequest\022\027\n\017collection_name\030\001 \001" - "(\t\022\026\n\016partition_name\030\002 \001(\t\022\021\n\trow_based\030" - "\003 \001(\010\022\r\n\005files\030\004 \003(\t\0222\n\007options\030\005 \003(\0132!." - "milvus.proto.common.KeyValuePair\"L\n\016Impo" - "rtResponse\022+\n\006status\030\001 \001(\0132\033.milvus.prot" - "o.common.Status\022\r\n\005tasks\030\002 \003(\003\"%\n\025GetImp" - "ortStateRequest\022\014\n\004task\030\001 \001(\003\"\314\001\n\026GetImp" - "ortStateResponse\022+\n\006status\030\001 \001(\0132\033.milvu" - "s.proto.common.Status\022/\n\005state\030\002 \001(\0162 .m" - "ilvus.proto.common.ImportState\022\021\n\trow_co" - "unt\030\003 \001(\003\022\017\n\007id_list\030\004 \003(\003\0220\n\005infos\030\005 \003(" - "\0132!.milvus.proto.common.KeyValuePair*!\n\010" - "ShowType\022\007\n\003All\020\000\022\014\n\010InMemory\020\001*>\n\017Place" - "holderType\022\010\n\004None\020\000\022\020\n\014BinaryVector\020d\022\017" - "\n\013FloatVector\020e2\307 \n\rMilvusService\022_\n\020Cre" - "ateCollection\022,.milvus.proto.milvus.Crea" - "teCollectionRequest\032\033.milvus.proto.commo" - "n.Status\"\000\022[\n\016DropCollection\022*.milvus.pr" - "oto.milvus.DropCollectionRequest\032\033.milvu" - "s.proto.common.Status\"\000\022_\n\rHasCollection" - "\022).milvus.proto.milvus.HasCollectionRequ" - "est\032!.milvus.proto.milvus.BoolResponse\"\000" - "\022[\n\016LoadCollection\022*.milvus.proto.milvus" - ".LoadCollectionRequest\032\033.milvus.proto.co" - "mmon.Status\"\000\022a\n\021ReleaseCollection\022-.mil" - "vus.proto.milvus.ReleaseCollectionReques" - "t\032\033.milvus.proto.common.Status\"\000\022w\n\022Desc" - "ribeCollection\022..milvus.proto.milvus.Des" - "cribeCollectionRequest\032/.milvus.proto.mi" - "lvus.DescribeCollectionResponse\"\000\022\206\001\n\027Ge" - "tCollectionStatistics\0223.milvus.proto.mil" - "vus.GetCollectionStatisticsRequest\0324.mil" - "vus.proto.milvus.GetCollectionStatistics" - "Response\"\000\022n\n\017ShowCollections\022+.milvus.p" - "roto.milvus.ShowCollectionsRequest\032,.mil" - "vus.proto.milvus.ShowCollectionsResponse" - "\"\000\022]\n\017CreatePartition\022+.milvus.proto.mil" - "vus.CreatePartitionRequest\032\033.milvus.prot" - "o.common.Status\"\000\022Y\n\rDropPartition\022).mil" - "vus.proto.milvus.DropPartitionRequest\032\033." - "milvus.proto.common.Status\"\000\022]\n\014HasParti" - "tion\022(.milvus.proto.milvus.HasPartitionR" - "equest\032!.milvus.proto.milvus.BoolRespons" - "e\"\000\022[\n\016LoadPartitions\022*.milvus.proto.mil" - "vus.LoadPartitionsRequest\032\033.milvus.proto" - ".common.Status\"\000\022a\n\021ReleasePartitions\022-." - "milvus.proto.milvus.ReleasePartitionsReq" - "uest\032\033.milvus.proto.common.Status\"\000\022\203\001\n\026" - "GetPartitionStatistics\0222.milvus.proto.mi" - "lvus.GetPartitionStatisticsRequest\0323.mil" - "vus.proto.milvus.GetPartitionStatisticsR" - "esponse\"\000\022k\n\016ShowPartitions\022*.milvus.pro" - "to.milvus.ShowPartitionsRequest\032+.milvus" - ".proto.milvus.ShowPartitionsResponse\"\000\022U" - "\n\013CreateAlias\022\'.milvus.proto.milvus.Crea" - "teAliasRequest\032\033.milvus.proto.common.Sta" - "tus\"\000\022Q\n\tDropAlias\022%.milvus.proto.milvus" - ".DropAliasRequest\032\033.milvus.proto.common." - "Status\"\000\022S\n\nAlterAlias\022&.milvus.proto.mi" - "lvus.AlterAliasRequest\032\033.milvus.proto.co" - "mmon.Status\"\000\022U\n\013CreateIndex\022\'.milvus.pr" - "oto.milvus.CreateIndexRequest\032\033.milvus.p" - "roto.common.Status\"\000\022h\n\rDescribeIndex\022)." - "milvus.proto.milvus.DescribeIndexRequest" - "\032*.milvus.proto.milvus.DescribeIndexResp" - "onse\"\000\022h\n\rGetIndexState\022).milvus.proto.m" - "ilvus.GetIndexStateRequest\032*.milvus.prot" - "o.milvus.GetIndexStateResponse\"\000\022\200\001\n\025Get" - "IndexBuildProgress\0221.milvus.proto.milvus" - ".GetIndexBuildProgressRequest\0322.milvus.p" - "roto.milvus.GetIndexBuildProgressRespons" - "e\"\000\022Q\n\tDropIndex\022%.milvus.proto.milvus.D" - "ropIndexRequest\032\033.milvus.proto.common.St" - "atus\"\000\022S\n\006Insert\022\".milvus.proto.milvus.I" - "nsertRequest\032#.milvus.proto.milvus.Mutat" - "ionResult\"\000\022S\n\006Delete\022\".milvus.proto.mil" - "vus.DeleteRequest\032#.milvus.proto.milvus." - "MutationResult\"\000\022R\n\006Search\022\".milvus.prot" - "o.milvus.SearchRequest\032\".milvus.proto.mi" - "lvus.SearchResults\"\000\022P\n\005Flush\022!.milvus.p" - "roto.milvus.FlushRequest\032\".milvus.proto." - "milvus.FlushResponse\"\000\022O\n\005Query\022!.milvus" - ".proto.milvus.QueryRequest\032!.milvus.prot" - "o.milvus.QueryResults\"\000\022d\n\014CalcDistance\022" - "(.milvus.proto.milvus.CalcDistanceReques" - "t\032(.milvus.proto.milvus.CalcDistanceResu" - "lts\"\000\022h\n\rGetFlushState\022).milvus.proto.mi" - "lvus.GetFlushStateRequest\032*.milvus.proto" - ".milvus.GetFlushStateResponse\"\000\022\211\001\n\030GetP" - "ersistentSegmentInfo\0224.milvus.proto.milv" - "us.GetPersistentSegmentInfoRequest\0325.mil" - "vus.proto.milvus.GetPersistentSegmentInf" - "oResponse\"\000\022z\n\023GetQuerySegmentInfo\022/.mil" - "vus.proto.milvus.GetQuerySegmentInfoRequ" - "est\0320.milvus.proto.milvus.GetQuerySegmen" - "tInfoResponse\"\000\022P\n\005Dummy\022!.milvus.proto." - "milvus.DummyRequest\032\".milvus.proto.milvu" - "s.DummyResponse\"\000\022e\n\014RegisterLink\022(.milv" - "us.proto.milvus.RegisterLinkRequest\032).mi" - "lvus.proto.milvus.RegisterLinkResponse\"\000" - "\022_\n\nGetMetrics\022&.milvus.proto.milvus.Get" - "MetricsRequest\032\'.milvus.proto.milvus.Get" - "MetricsResponse\"\000\022U\n\013LoadBalance\022\'.milvu" - "s.proto.milvus.LoadBalanceRequest\032\033.milv" - "us.proto.common.Status\"\000\022w\n\022GetCompactio" - "nState\022..milvus.proto.milvus.GetCompacti" - "onStateRequest\032/.milvus.proto.milvus.Get" - "CompactionStateResponse\"\000\022q\n\020ManualCompa" - "ction\022,.milvus.proto.milvus.ManualCompac" - "tionRequest\032-.milvus.proto.milvus.Manual" - "CompactionResponse\"\000\022\200\001\n\033GetCompactionSt" - "ateWithPlans\022..milvus.proto.milvus.GetCo" - "mpactionPlansRequest\032/.milvus.proto.milv" - "us.GetCompactionPlansResponse\"\000\022S\n\006Impor" - "t\022\".milvus.proto.milvus.ImportRequest\032#." - "milvus.proto.milvus.ImportResponse\"\000\022k\n\016" - "GetImportState\022*.milvus.proto.milvus.Get" - "ImportStateRequest\032+.milvus.proto.milvus" - ".GetImportStateResponse\"\0002u\n\014ProxyServic" - "e\022e\n\014RegisterLink\022(.milvus.proto.milvus." - "RegisterLinkRequest\032).milvus.proto.milvu" - "s.RegisterLinkResponse\"\000B5Z3github.com/m" - "ilvus-io/milvus/internal/proto/milvuspbb" - "\006proto3" + "s\022/\n\005state\030\002 \001(\0162 .milvus.proto.common.I" + "mportState\022\021\n\trow_count\030\003 \001(\003\022\017\n\007id_list" + "\030\004 \003(\003\0220\n\005infos\030\005 \003(\0132!.milvus.proto.com" + "mon.KeyValuePair*!\n\010ShowType\022\007\n\003All\020\000\022\014\n" + "\010InMemory\020\001*>\n\017PlaceholderType\022\010\n\004None\020\000" + "\022\020\n\014BinaryVector\020d\022\017\n\013FloatVector\020e2\307 \n\r" + "MilvusService\022_\n\020CreateCollection\022,.milv" + "us.proto.milvus.CreateCollectionRequest\032" + "\033.milvus.proto.common.Status\"\000\022[\n\016DropCo" + "llection\022*.milvus.proto.milvus.DropColle" + "ctionRequest\032\033.milvus.proto.common.Statu" + "s\"\000\022_\n\rHasCollection\022).milvus.proto.milv" + "us.HasCollectionRequest\032!.milvus.proto.m" + "ilvus.BoolResponse\"\000\022[\n\016LoadCollection\022*" + ".milvus.proto.milvus.LoadCollectionReque" + "st\032\033.milvus.proto.common.Status\"\000\022a\n\021Rel" + "easeCollection\022-.milvus.proto.milvus.Rel" + "easeCollectionRequest\032\033.milvus.proto.com" + "mon.Status\"\000\022w\n\022DescribeCollection\022..mil" + "vus.proto.milvus.DescribeCollectionReque" + "st\032/.milvus.proto.milvus.DescribeCollect" + "ionResponse\"\000\022\206\001\n\027GetCollectionStatistic" + "s\0223.milvus.proto.milvus.GetCollectionSta" + "tisticsRequest\0324.milvus.proto.milvus.Get" + "CollectionStatisticsResponse\"\000\022n\n\017ShowCo" + "llections\022+.milvus.proto.milvus.ShowColl" + "ectionsRequest\032,.milvus.proto.milvus.Sho" + "wCollectionsResponse\"\000\022]\n\017CreatePartitio" + "n\022+.milvus.proto.milvus.CreatePartitionR" + "equest\032\033.milvus.proto.common.Status\"\000\022Y\n" + "\rDropPartition\022).milvus.proto.milvus.Dro" + "pPartitionRequest\032\033.milvus.proto.common." + "Status\"\000\022]\n\014HasPartition\022(.milvus.proto." + "milvus.HasPartitionRequest\032!.milvus.prot" + "o.milvus.BoolResponse\"\000\022[\n\016LoadPartition" + "s\022*.milvus.proto.milvus.LoadPartitionsRe" + "quest\032\033.milvus.proto.common.Status\"\000\022a\n\021" + "ReleasePartitions\022-.milvus.proto.milvus." + "ReleasePartitionsRequest\032\033.milvus.proto." + "common.Status\"\000\022\203\001\n\026GetPartitionStatisti" + "cs\0222.milvus.proto.milvus.GetPartitionSta" + "tisticsRequest\0323.milvus.proto.milvus.Get" + "PartitionStatisticsResponse\"\000\022k\n\016ShowPar" + "titions\022*.milvus.proto.milvus.ShowPartit" + "ionsRequest\032+.milvus.proto.milvus.ShowPa" + "rtitionsResponse\"\000\022U\n\013CreateAlias\022\'.milv" + "us.proto.milvus.CreateAliasRequest\032\033.mil" + "vus.proto.common.Status\"\000\022Q\n\tDropAlias\022%" + ".milvus.proto.milvus.DropAliasRequest\032\033." + "milvus.proto.common.Status\"\000\022S\n\nAlterAli" + "as\022&.milvus.proto.milvus.AlterAliasReque" + "st\032\033.milvus.proto.common.Status\"\000\022U\n\013Cre" + "ateIndex\022\'.milvus.proto.milvus.CreateInd" + "exRequest\032\033.milvus.proto.common.Status\"\000" + "\022h\n\rDescribeIndex\022).milvus.proto.milvus." + "DescribeIndexRequest\032*.milvus.proto.milv" + "us.DescribeIndexResponse\"\000\022h\n\rGetIndexSt" + "ate\022).milvus.proto.milvus.GetIndexStateR" + "equest\032*.milvus.proto.milvus.GetIndexSta" + "teResponse\"\000\022\200\001\n\025GetIndexBuildProgress\0221" + ".milvus.proto.milvus.GetIndexBuildProgre" + "ssRequest\0322.milvus.proto.milvus.GetIndex" + "BuildProgressResponse\"\000\022Q\n\tDropIndex\022%.m" + "ilvus.proto.milvus.DropIndexRequest\032\033.mi" + "lvus.proto.common.Status\"\000\022S\n\006Insert\022\".m" + "ilvus.proto.milvus.InsertRequest\032#.milvu" + "s.proto.milvus.MutationResult\"\000\022S\n\006Delet" + "e\022\".milvus.proto.milvus.DeleteRequest\032#." + "milvus.proto.milvus.MutationResult\"\000\022R\n\006" + "Search\022\".milvus.proto.milvus.SearchReque" + "st\032\".milvus.proto.milvus.SearchResults\"\000" + "\022P\n\005Flush\022!.milvus.proto.milvus.FlushReq" + "uest\032\".milvus.proto.milvus.FlushResponse" + "\"\000\022O\n\005Query\022!.milvus.proto.milvus.QueryR" + "equest\032!.milvus.proto.milvus.QueryResult" + "s\"\000\022d\n\014CalcDistance\022(.milvus.proto.milvu" + "s.CalcDistanceRequest\032(.milvus.proto.mil" + "vus.CalcDistanceResults\"\000\022h\n\rGetFlushSta" + "te\022).milvus.proto.milvus.GetFlushStateRe" + "quest\032*.milvus.proto.milvus.GetFlushStat" + "eResponse\"\000\022\211\001\n\030GetPersistentSegmentInfo" + "\0224.milvus.proto.milvus.GetPersistentSegm" + "entInfoRequest\0325.milvus.proto.milvus.Get" + "PersistentSegmentInfoResponse\"\000\022z\n\023GetQu" + "erySegmentInfo\022/.milvus.proto.milvus.Get" + "QuerySegmentInfoRequest\0320.milvus.proto.m" + "ilvus.GetQuerySegmentInfoResponse\"\000\022P\n\005D" + "ummy\022!.milvus.proto.milvus.DummyRequest\032" + "\".milvus.proto.milvus.DummyResponse\"\000\022e\n" + "\014RegisterLink\022(.milvus.proto.milvus.Regi" + "sterLinkRequest\032).milvus.proto.milvus.Re" + "gisterLinkResponse\"\000\022_\n\nGetMetrics\022&.mil" + "vus.proto.milvus.GetMetricsRequest\032\'.mil" + "vus.proto.milvus.GetMetricsResponse\"\000\022U\n" + "\013LoadBalance\022\'.milvus.proto.milvus.LoadB" + "alanceRequest\032\033.milvus.proto.common.Stat" + "us\"\000\022w\n\022GetCompactionState\022..milvus.prot" + "o.milvus.GetCompactionStateRequest\032/.mil" + "vus.proto.milvus.GetCompactionStateRespo" + "nse\"\000\022q\n\020ManualCompaction\022,.milvus.proto" + ".milvus.ManualCompactionRequest\032-.milvus" + ".proto.milvus.ManualCompactionResponse\"\000" + "\022\200\001\n\033GetCompactionStateWithPlans\022..milvu" + "s.proto.milvus.GetCompactionPlansRequest" + "\032/.milvus.proto.milvus.GetCompactionPlan" + "sResponse\"\000\022S\n\006Import\022\".milvus.proto.mil" + "vus.ImportRequest\032#.milvus.proto.milvus." + "ImportResponse\"\000\022k\n\016GetImportState\022*.mil" + "vus.proto.milvus.GetImportStateRequest\032+" + ".milvus.proto.milvus.GetImportStateRespo" + "nse\"\0002u\n\014ProxyService\022e\n\014RegisterLink\022(." + "milvus.proto.milvus.RegisterLinkRequest\032" + ").milvus.proto.milvus.RegisterLinkRespon" + "se\"\000B5Z3github.com/milvus-io/milvus/inte" + "rnal/proto/milvuspbb\006proto3" ; static const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable*const descriptor_table_milvus_2eproto_deps[2] = { &::descriptor_table_common_2eproto, @@ -2957,7 +2958,7 @@ static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_mil static ::PROTOBUF_NAMESPACE_ID::internal::once_flag descriptor_table_milvus_2eproto_once; static bool descriptor_table_milvus_2eproto_initialized = false; const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable descriptor_table_milvus_2eproto = { - &descriptor_table_milvus_2eproto_initialized, descriptor_table_protodef_milvus_2eproto, "milvus.proto", 15567, + &descriptor_table_milvus_2eproto_initialized, descriptor_table_protodef_milvus_2eproto, "milvus.proto", 15587, &descriptor_table_milvus_2eproto_once, descriptor_table_milvus_2eproto_sccs, descriptor_table_milvus_2eproto_deps, 81, 2, schemas, file_default_instances, TableStruct_milvus_2eproto::offsets, file_level_metadata_milvus_2eproto, 81, file_level_enum_descriptors_milvus_2eproto, file_level_service_descriptors_milvus_2eproto, @@ -16115,6 +16116,10 @@ CreateIndexRequest::CreateIndexRequest(const CreateIndexRequest& from) if (!from.field_name().empty()) { field_name_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.field_name_); } + index_name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + if (!from.index_name().empty()) { + index_name_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.index_name_); + } if (from.has_base()) { base_ = new ::milvus::proto::common::MsgBase(*from.base_); } else { @@ -16128,6 +16133,7 @@ void CreateIndexRequest::SharedCtor() { db_name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); collection_name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); field_name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + index_name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); base_ = nullptr; } @@ -16140,6 +16146,7 @@ void CreateIndexRequest::SharedDtor() { db_name_.DestroyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); collection_name_.DestroyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); field_name_.DestroyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + index_name_.DestroyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); if (this != internal_default_instance()) delete base_; } @@ -16162,6 +16169,7 @@ void CreateIndexRequest::Clear() { db_name_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); collection_name_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); field_name_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + index_name_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); if (GetArenaNoVirtual() == nullptr && base_ != nullptr) { delete base_; } @@ -16217,6 +16225,13 @@ const char* CreateIndexRequest::_InternalParse(const char* ptr, ::PROTOBUF_NAMES } while (::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<::PROTOBUF_NAMESPACE_ID::uint8>(ptr) == 42); } else goto handle_unusual; continue; + // string index_name = 6; + case 6: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 50)) { + ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParserUTF8(mutable_index_name(), ptr, ctx, "milvus.proto.milvus.CreateIndexRequest.index_name"); + CHK_(ptr); + } else goto handle_unusual; + continue; default: { handle_unusual: if ((tag & 7) == 4 || tag == 0) { @@ -16314,6 +16329,21 @@ bool CreateIndexRequest::MergePartialFromCodedStream( break; } + // string index_name = 6; + case 6: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (50 & 0xFF)) { + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadString( + input, this->mutable_index_name())); + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->index_name().data(), static_cast(this->index_name().length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, + "milvus.proto.milvus.CreateIndexRequest.index_name")); + } else { + goto handle_unusual; + } + break; + } + default: { handle_unusual: if (tag == 0) { @@ -16386,6 +16416,16 @@ void CreateIndexRequest::SerializeWithCachedSizes( output); } + // string index_name = 6; + if (this->index_name().size() > 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->index_name().data(), static_cast(this->index_name().length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, + "milvus.proto.milvus.CreateIndexRequest.index_name"); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteStringMaybeAliased( + 6, this->index_name(), output); + } + if (_internal_metadata_.have_unknown_fields()) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFields( _internal_metadata_.unknown_fields(), output); @@ -16447,6 +16487,17 @@ void CreateIndexRequest::SerializeWithCachedSizes( 5, this->extra_params(static_cast(i)), target); } + // string index_name = 6; + if (this->index_name().size() > 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->index_name().data(), static_cast(this->index_name().length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, + "milvus.proto.milvus.CreateIndexRequest.index_name"); + target = + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteStringToArray( + 6, this->index_name(), target); + } + if (_internal_metadata_.have_unknown_fields()) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields(), target); @@ -16500,6 +16551,13 @@ size_t CreateIndexRequest::ByteSizeLong() const { this->field_name()); } + // string index_name = 6; + if (this->index_name().size() > 0) { + total_size += 1 + + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( + this->index_name()); + } + // .milvus.proto.common.MsgBase base = 1; if (this->has_base()) { total_size += 1 + @@ -16547,6 +16605,10 @@ void CreateIndexRequest::MergeFrom(const CreateIndexRequest& from) { field_name_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.field_name_); } + if (from.index_name().size() > 0) { + + index_name_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.index_name_); + } if (from.has_base()) { mutable_base()->::milvus::proto::common::MsgBase::MergeFrom(from.base()); } @@ -16580,6 +16642,8 @@ void CreateIndexRequest::InternalSwap(CreateIndexRequest* other) { GetArenaNoVirtual()); field_name_.Swap(&other->field_name_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArenaNoVirtual()); + index_name_.Swap(&other->index_name_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + GetArenaNoVirtual()); swap(base_, other->base_); } diff --git a/internal/core/src/pb/milvus.pb.h b/internal/core/src/pb/milvus.pb.h index 4d7ed2ce9a676fb04201388ca3cc5d7c03e89c5f..37a84d69cc89901b389b50a523cbd6763a73c534 100644 --- a/internal/core/src/pb/milvus.pb.h +++ b/internal/core/src/pb/milvus.pb.h @@ -5531,6 +5531,7 @@ class CreateIndexRequest : kDbNameFieldNumber = 2, kCollectionNameFieldNumber = 3, kFieldNameFieldNumber = 4, + kIndexNameFieldNumber = 6, kBaseFieldNumber = 1, }; // repeated .milvus.proto.common.KeyValuePair extra_params = 5; @@ -5577,6 +5578,17 @@ class CreateIndexRequest : std::string* release_field_name(); void set_allocated_field_name(std::string* field_name); + // string index_name = 6; + void clear_index_name(); + const std::string& index_name() const; + void set_index_name(const std::string& value); + void set_index_name(std::string&& value); + void set_index_name(const char* value); + void set_index_name(const char* value, size_t size); + std::string* mutable_index_name(); + std::string* release_index_name(); + void set_allocated_index_name(std::string* index_name); + // .milvus.proto.common.MsgBase base = 1; bool has_base() const; void clear_base(); @@ -5594,6 +5606,7 @@ class CreateIndexRequest : ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr db_name_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr collection_name_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr field_name_; + ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr index_name_; ::milvus::proto::common::MsgBase* base_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; friend struct ::TableStruct_milvus_2eproto; @@ -18997,6 +19010,57 @@ CreateIndexRequest::extra_params() const { return extra_params_; } +// string index_name = 6; +inline void CreateIndexRequest::clear_index_name() { + index_name_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} +inline const std::string& CreateIndexRequest::index_name() const { + // @@protoc_insertion_point(field_get:milvus.proto.milvus.CreateIndexRequest.index_name) + return index_name_.GetNoArena(); +} +inline void CreateIndexRequest::set_index_name(const std::string& value) { + + index_name_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value); + // @@protoc_insertion_point(field_set:milvus.proto.milvus.CreateIndexRequest.index_name) +} +inline void CreateIndexRequest::set_index_name(std::string&& value) { + + index_name_.SetNoArena( + &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); + // @@protoc_insertion_point(field_set_rvalue:milvus.proto.milvus.CreateIndexRequest.index_name) +} +inline void CreateIndexRequest::set_index_name(const char* value) { + GOOGLE_DCHECK(value != nullptr); + + index_name_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); + // @@protoc_insertion_point(field_set_char:milvus.proto.milvus.CreateIndexRequest.index_name) +} +inline void CreateIndexRequest::set_index_name(const char* value, size_t size) { + + index_name_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + ::std::string(reinterpret_cast(value), size)); + // @@protoc_insertion_point(field_set_pointer:milvus.proto.milvus.CreateIndexRequest.index_name) +} +inline std::string* CreateIndexRequest::mutable_index_name() { + + // @@protoc_insertion_point(field_mutable:milvus.proto.milvus.CreateIndexRequest.index_name) + return index_name_.MutableNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} +inline std::string* CreateIndexRequest::release_index_name() { + // @@protoc_insertion_point(field_release:milvus.proto.milvus.CreateIndexRequest.index_name) + + return index_name_.ReleaseNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} +inline void CreateIndexRequest::set_allocated_index_name(std::string* index_name) { + if (index_name != nullptr) { + + } else { + + } + index_name_.SetAllocatedNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), index_name); + // @@protoc_insertion_point(field_set_allocated:milvus.proto.milvus.CreateIndexRequest.index_name) +} + // ------------------------------------------------------------------- // DescribeIndexRequest diff --git a/internal/core/unittest/CMakeLists.txt b/internal/core/unittest/CMakeLists.txt index 88fa7c0302075811a11f89c6664320ebdbeab4fc..1d95c32568f92403c551451e120cccde8b3a5122 100644 --- a/internal/core/unittest/CMakeLists.txt +++ b/internal/core/unittest/CMakeLists.txt @@ -15,6 +15,7 @@ include_directories(${CMAKE_HOME_DIRECTORY}/src/thirdparty) add_definitions(-DMILVUS_TEST_SEGCORE_YAML_PATH="${CMAKE_SOURCE_DIR}/unittest/test_utils/test_segcore.yaml") if (LINUX) + # TODO: better to use ls/find pattern set(MILVUS_TEST_FILES init_gtest.cpp test_binary.cpp @@ -38,10 +39,14 @@ if (LINUX) test_conf_adapter_mgr.cpp test_similarity_corelation.cpp test_utils.cpp + test_scalar_index_creator.cpp + test_index_c_api.cpp ) # check if memory leak exists in index builder set(INDEX_BUILDER_TEST_FILES test_index_wrapper.cpp + test_scalar_index_creator.cpp + test_index_c_api.cpp ) add_executable(index_builder_test diff --git a/internal/core/unittest/test_index_c_api.cpp b/internal/core/unittest/test_index_c_api.cpp new file mode 100644 index 0000000000000000000000000000000000000000..eda507b52446c5c2f323451c28736a83a2dce7d2 --- /dev/null +++ b/internal/core/unittest/test_index_c_api.cpp @@ -0,0 +1,289 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License + +#include +#include +#include +#include +#include +#include +#include +#include +#include "pb/index_cgo_msg.pb.h" + +#include "indexbuilder/VecIndexCreator.h" +#include "indexbuilder/index_c.h" +#include "indexbuilder/utils.h" +#include "test_utils/DataGen.h" +#include "test_utils/indexbuilder_test_utils.h" +#include "indexbuilder/ScalarIndexCreator.h" +#include "indexbuilder/IndexFactory.h" + +constexpr int NB = 10; + +TEST(FloatVecIndex, All) { + auto index_type = milvus::knowhere::IndexEnum::INDEX_FAISS_IVFPQ; + auto metric_type = milvus::knowhere::Metric::L2; + indexcgo::TypeParams type_params; + indexcgo::IndexParams index_params; + std::tie(type_params, index_params) = generate_params(index_type, metric_type); + std::string type_params_str, index_params_str; + bool ok; + ok = google::protobuf::TextFormat::PrintToString(type_params, &type_params_str); + assert(ok); + ok = google::protobuf::TextFormat::PrintToString(index_params, &index_params_str); + assert(ok); + auto dataset = GenDataset(NB, metric_type, false); + auto xb_data = dataset.get_col(0); + + DataType dtype = FloatVector; + CIndex index; + CStatus status; + CBinarySet binary_set; + CIndex copy_index; + + { + status = CreateIndex(dtype, type_params_str.c_str(), index_params_str.c_str(), &index); + ASSERT_EQ(Success, status.error_code); + } + { + status = BuildFloatVecIndex(index, NB * DIM, xb_data.data()); + ASSERT_EQ(Success, status.error_code); + } + { + status = SerializeIndexToBinarySet(index, &binary_set); + ASSERT_EQ(Success, status.error_code); + } + { + status = CreateIndex(dtype, type_params_str.c_str(), index_params_str.c_str(), ©_index); + ASSERT_EQ(Success, status.error_code); + } + { + status = LoadIndexFromBinarySet(copy_index, binary_set); + ASSERT_EQ(Success, status.error_code); + } + { + status = DeleteIndex(index); + ASSERT_EQ(Success, status.error_code); + } + { + status = DeleteIndex(copy_index); + ASSERT_EQ(Success, status.error_code); + } +} + +TEST(BinaryVecIndex, All) { + auto index_type = milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT; + auto metric_type = milvus::knowhere::Metric::JACCARD; + indexcgo::TypeParams type_params; + indexcgo::IndexParams index_params; + std::tie(type_params, index_params) = generate_params(index_type, metric_type); + std::string type_params_str, index_params_str; + bool ok; + ok = google::protobuf::TextFormat::PrintToString(type_params, &type_params_str); + assert(ok); + ok = google::protobuf::TextFormat::PrintToString(index_params, &index_params_str); + assert(ok); + auto dataset = GenDataset(NB, metric_type, true); + auto xb_data = dataset.get_col(0); + + DataType dtype = BinaryVector; + CIndex index; + CStatus status; + CBinarySet binary_set; + CIndex copy_index; + + { + status = CreateIndex(dtype, type_params_str.c_str(), index_params_str.c_str(), &index); + ASSERT_EQ(Success, status.error_code); + } + { + status = BuildBinaryVecIndex(index, NB * DIM / 8, xb_data.data()); + ASSERT_EQ(Success, status.error_code); + } + { + status = SerializeIndexToBinarySet(index, &binary_set); + ASSERT_EQ(Success, status.error_code); + } + { + status = CreateIndex(dtype, type_params_str.c_str(), index_params_str.c_str(), ©_index); + ASSERT_EQ(Success, status.error_code); + } + { + status = LoadIndexFromBinarySet(copy_index, binary_set); + ASSERT_EQ(Success, status.error_code); + } + { + status = DeleteIndex(index); + ASSERT_EQ(Success, status.error_code); + } + { + status = DeleteIndex(copy_index); + ASSERT_EQ(Success, status.error_code); + } +} + +TEST(CBoolIndexTest, All) { + schemapb::BoolArray half; + milvus::knowhere::DatasetPtr half_ds; + + for (size_t i = 0; i < NB; i++) { + *(half.mutable_data()->Add()) = (i % 2) == 0; + } + half_ds = GenDsFromPB(half); + + auto params = GenBoolParams(); + for (const auto& tp : params) { + auto type_params = tp.first; + auto index_params = tp.second; + auto type_params_str = generate_type_params(type_params); + auto index_params_str = generate_index_params(index_params); + + DataType dtype = Bool; + CIndex index; + CStatus status; + CBinarySet binary_set; + CIndex copy_index; + + { + status = CreateIndex(dtype, type_params_str.c_str(), index_params_str.c_str(), &index); + ASSERT_EQ(Success, status.error_code); + } + { + status = BuildScalarIndex(index, half_ds->Get(milvus::knowhere::meta::ROWS), + half_ds->Get(milvus::knowhere::meta::TENSOR)); + ASSERT_EQ(Success, status.error_code); + } + { + status = SerializeIndexToBinarySet(index, &binary_set); + ASSERT_EQ(Success, status.error_code); + } + { + status = CreateIndex(dtype, type_params_str.c_str(), index_params_str.c_str(), ©_index); + ASSERT_EQ(Success, status.error_code); + } + { + status = LoadIndexFromBinarySet(copy_index, binary_set); + ASSERT_EQ(Success, status.error_code); + } + { + status = DeleteIndex(index); + ASSERT_EQ(Success, status.error_code); + } + { + status = DeleteIndex(copy_index); + ASSERT_EQ(Success, status.error_code); + } + } + + delete[](char*) half_ds->Get(milvus::knowhere::meta::TENSOR); +} + +// TODO: more scalar type. +TEST(CInt64IndexTest, All) { + auto arr = GenArr(NB); + + auto params = GenParams(); + for (const auto& tp : params) { + auto type_params = tp.first; + auto index_params = tp.second; + auto type_params_str = generate_type_params(type_params); + auto index_params_str = generate_index_params(index_params); + + DataType dtype = Int64; + CIndex index; + CStatus status; + CBinarySet binary_set; + CIndex copy_index; + + { + status = CreateIndex(dtype, type_params_str.c_str(), index_params_str.c_str(), &index); + ASSERT_EQ(Success, status.error_code); + } + { + status = BuildScalarIndex(index, arr.size(), arr.data()); + ASSERT_EQ(Success, status.error_code); + } + { + status = SerializeIndexToBinarySet(index, &binary_set); + ASSERT_EQ(Success, status.error_code); + } + { + status = CreateIndex(dtype, type_params_str.c_str(), index_params_str.c_str(), ©_index); + ASSERT_EQ(Success, status.error_code); + } + { + status = LoadIndexFromBinarySet(copy_index, binary_set); + ASSERT_EQ(Success, status.error_code); + } + { + status = DeleteIndex(index); + ASSERT_EQ(Success, status.error_code); + } + { + status = DeleteIndex(copy_index); + ASSERT_EQ(Success, status.error_code); + } + } +} + +TEST(CStringIndexTest, All) { + auto strs = GenStrArr(NB); + schemapb::StringArray str_arr; + *str_arr.mutable_data() = {strs.begin(), strs.end()}; + auto str_ds = GenDsFromPB(str_arr); + + auto params = GenStringParams(); + for (const auto& tp : params) { + auto type_params = tp.first; + auto index_params = tp.second; + auto type_params_str = generate_type_params(type_params); + auto index_params_str = generate_index_params(index_params); + + DataType dtype = String; + CIndex index; + CStatus status; + CBinarySet binary_set; + CIndex copy_index; + + { + status = CreateIndex(dtype, type_params_str.c_str(), index_params_str.c_str(), &index); + ASSERT_EQ(Success, status.error_code); + } + { + status = BuildScalarIndex(index, str_ds->Get(milvus::knowhere::meta::ROWS), + str_ds->Get(milvus::knowhere::meta::TENSOR)); + ASSERT_EQ(Success, status.error_code); + } + { + status = SerializeIndexToBinarySet(index, &binary_set); + ASSERT_EQ(Success, status.error_code); + } + { + status = CreateIndex(dtype, type_params_str.c_str(), index_params_str.c_str(), ©_index); + ASSERT_EQ(Success, status.error_code); + } + { + status = LoadIndexFromBinarySet(copy_index, binary_set); + ASSERT_EQ(Success, status.error_code); + } + { + status = DeleteIndex(index); + ASSERT_EQ(Success, status.error_code); + } + { + status = DeleteIndex(copy_index); + ASSERT_EQ(Success, status.error_code); + } + } + + delete[](char*) str_ds->Get(milvus::knowhere::meta::TENSOR); +} diff --git a/internal/core/unittest/test_index_wrapper.cpp b/internal/core/unittest/test_index_wrapper.cpp index 4a8c2de029e9f4de1f3b9a4373954da1839322a4..5101c496977e06989ca6c2ecad5fe5bdd41aad17 100644 --- a/internal/core/unittest/test_index_wrapper.cpp +++ b/internal/core/unittest/test_index_wrapper.cpp @@ -18,7 +18,7 @@ #include #include -#include "indexbuilder/IndexWrapper.h" +#include "indexbuilder/VecIndexCreator.h" #include "indexbuilder/index_c.h" #include "indexbuilder/utils.h" #include "pb/index_cgo_msg.pb.h" @@ -191,7 +191,7 @@ TEST(BINFLAT, Build) { } void -print_query_result(const std::unique_ptr& result) { +print_query_result(const std::unique_ptr& result) { for (auto i = 0; i < result->nq; i++) { printf("result of %dth query:\n", i); for (auto j = 0; j < result->topk; j++) { @@ -228,7 +228,7 @@ TEST(BinIVFFlat, Build_and_Query) { auto hit_ids = result->Get(milvus::knowhere::meta::IDS); auto distances = result->Get(milvus::knowhere::meta::DISTANCE); - auto query_res = std::make_unique(); + auto query_res = std::make_unique(); query_res->nq = nq; query_res->topk = topk; query_res->ids.resize(nq * topk); @@ -275,7 +275,7 @@ TEST(PQWrapper, Build) { auto xb_data = dataset.get_col(0); auto xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_data.data()); auto index = - std::make_unique(type_params_str.c_str(), index_params_str.c_str()); + std::make_unique(type_params_str.c_str(), index_params_str.c_str()); ASSERT_NO_THROW(index->BuildWithoutIds(xb_dataset)); } @@ -295,7 +295,7 @@ TEST(IVFFLATNMWrapper, Build) { auto xb_data = dataset.get_col(0); auto xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_data.data()); auto index = - std::make_unique(type_params_str.c_str(), index_params_str.c_str()); + std::make_unique(type_params_str.c_str(), index_params_str.c_str()); ASSERT_NO_THROW(index->BuildWithoutIds(xb_dataset)); } @@ -316,15 +316,22 @@ TEST(IVFFLATNMWrapper, Codec) { auto xb_data = dataset.get_col(0); auto xb_dataset = milvus::knowhere::GenDataset(flat_nb, DIM, xb_data.data()); auto index_wrapper = - std::make_unique(type_params_str.c_str(), index_params_str.c_str()); + std::make_unique(type_params_str.c_str(), index_params_str.c_str()); ASSERT_NO_THROW(index_wrapper->BuildWithoutIds(xb_dataset)); - auto binary = index_wrapper->Serialize(); + auto binary_set = index_wrapper->Serialize(); auto copy_index_wrapper = - std::make_unique(type_params_str.c_str(), index_params_str.c_str()); - ASSERT_NO_THROW(copy_index_wrapper->Load(binary->data.data(), binary->data.size())); + std::make_unique(type_params_str.c_str(), index_params_str.c_str()); + + ASSERT_NO_THROW(copy_index_wrapper->Load(binary_set)); ASSERT_EQ(copy_index_wrapper->dim(), copy_index_wrapper->dim()); - auto copy_binary = copy_index_wrapper->Serialize(); + + auto copy_binary_set = copy_index_wrapper->Serialize(); + ASSERT_EQ(binary_set.binary_map_.size(), copy_binary_set.binary_map_.size()); + + for (const auto& [k, v] : binary_set.binary_map_) { + ASSERT_TRUE(copy_binary_set.Contains(k)); + } } TEST(BinFlatWrapper, Build) { @@ -345,7 +352,7 @@ TEST(BinFlatWrapper, Build) { std::iota(ids.begin(), ids.end(), 0); auto xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_data.data()); auto index = - std::make_unique(type_params_str.c_str(), index_params_str.c_str()); + std::make_unique(type_params_str.c_str(), index_params_str.c_str()); ASSERT_NO_THROW(index->BuildWithoutIds(xb_dataset)); // ASSERT_NO_THROW(index->BuildWithIds(xb_dataset)); } @@ -368,7 +375,7 @@ TEST(BinIdMapWrapper, Build) { std::iota(ids.begin(), ids.end(), 0); auto xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_data.data()); auto index = - std::make_unique(type_params_str.c_str(), index_params_str.c_str()); + std::make_unique(type_params_str.c_str(), index_params_str.c_str()); ASSERT_NO_THROW(index->BuildWithoutIds(xb_dataset)); // ASSERT_NO_THROW(index->BuildWithIds(xb_dataset)); } @@ -399,48 +406,50 @@ INSTANTIATE_TEST_CASE_P( TEST_P(IndexWrapperTest, Constructor) { auto index = - std::make_unique(type_params_str.c_str(), index_params_str.c_str()); + std::make_unique(type_params_str.c_str(), index_params_str.c_str()); } TEST_P(IndexWrapperTest, Dim) { auto index = - std::make_unique(type_params_str.c_str(), index_params_str.c_str()); + std::make_unique(type_params_str.c_str(), index_params_str.c_str()); ASSERT_EQ(index->dim(), DIM); } TEST_P(IndexWrapperTest, BuildWithoutIds) { auto index = - std::make_unique(type_params_str.c_str(), index_params_str.c_str()); + std::make_unique(type_params_str.c_str(), index_params_str.c_str()); ASSERT_NO_THROW(index->BuildWithoutIds(xb_dataset)); } TEST_P(IndexWrapperTest, Codec) { auto index_wrapper = - std::make_unique(type_params_str.c_str(), index_params_str.c_str()); + std::make_unique(type_params_str.c_str(), index_params_str.c_str()); ASSERT_NO_THROW(index_wrapper->BuildWithoutIds(xb_dataset)); - auto binary = index_wrapper->Serialize(); + auto binary_set = index_wrapper->Serialize(); auto copy_index_wrapper = - std::make_unique(type_params_str.c_str(), index_params_str.c_str()); - ASSERT_NO_THROW(copy_index_wrapper->Load(binary->data.data(), binary->data.size())); + std::make_unique(type_params_str.c_str(), index_params_str.c_str()); + + ASSERT_NO_THROW(copy_index_wrapper->Load(binary_set)); ASSERT_EQ(copy_index_wrapper->dim(), copy_index_wrapper->dim()); - auto copy_binary = copy_index_wrapper->Serialize(); - if (!milvus::indexbuilder::is_in_nm_list(index_type)) { - // binary may be not same due to uncertain internal map order - ASSERT_EQ(binary->data.size(), copy_binary->data.size()); - ASSERT_EQ(binary->data, copy_binary->data); + + auto copy_binary_set = copy_index_wrapper->Serialize(); + ASSERT_EQ(binary_set.binary_map_.size(), copy_binary_set.binary_map_.size()); + + for (const auto& [k, v] : binary_set.binary_map_) { + ASSERT_TRUE(copy_binary_set.Contains(k)); } } TEST_P(IndexWrapperTest, Query) { auto index_wrapper = - std::make_unique(type_params_str.c_str(), index_params_str.c_str()); + std::make_unique(type_params_str.c_str(), index_params_str.c_str()); index_wrapper->BuildWithoutIds(xb_dataset); - std::unique_ptr query_result = index_wrapper->Query(xq_dataset); + std::unique_ptr query_result = index_wrapper->Query(xq_dataset); ASSERT_EQ(query_result->topk, K); ASSERT_EQ(query_result->nq, NQ); ASSERT_EQ(query_result->distances.size(), query_result->topk * query_result->nq); diff --git a/internal/core/unittest/test_scalar_index_creator.cpp b/internal/core/unittest/test_scalar_index_creator.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b0e8aa7910d7a5c1c080b4de9d7de95dd9b31f48 --- /dev/null +++ b/internal/core/unittest/test_scalar_index_creator.cpp @@ -0,0 +1,588 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License + +#include +#include +#include +#include +#include +#include +#include +#include +#include "pb/index_cgo_msg.pb.h" + +#define private public + +#include "indexbuilder/VecIndexCreator.h" +#include "indexbuilder/index_c.h" +#include "indexbuilder/utils.h" +#include "test_utils/DataGen.h" +#include "test_utils/indexbuilder_test_utils.h" +#include "indexbuilder/ScalarIndexCreator.h" +#include "indexbuilder/IndexFactory.h" + +TEST(Dummy, Aha) { + std::cout << "aha" << std::endl; +} + +constexpr int64_t nb = 100; +namespace indexcgo = milvus::proto::indexcgo; +namespace schemapb = milvus::proto::schema; +using milvus::indexbuilder::MapParams; +using milvus::indexbuilder::ScalarIndexCreator; +using milvus::knowhere::scalar::OperatorType; +using ScalarTestParams = std::pair; + +namespace { +template +inline void +assert_in(const std::unique_ptr>& creator, const std::vector& arr) { + // hard to compare floating point value. + if (std::is_floating_point_v) { + return; + } + + auto bitset1 = creator->index_->In(arr.size(), arr.data()); + ASSERT_EQ(arr.size(), bitset1->size()); + ASSERT_TRUE(bitset1->any()); + auto test = std::make_unique(arr[arr.size() - 1] + 1); + auto bitset2 = creator->index_->In(1, test.get()); + ASSERT_EQ(arr.size(), bitset2->size()); + ASSERT_TRUE(bitset2->none()); +} + +template +inline void +assert_not_in(const std::unique_ptr>& creator, const std::vector& arr) { + auto bitset1 = creator->index_->NotIn(arr.size(), arr.data()); + ASSERT_EQ(arr.size(), bitset1->size()); + ASSERT_TRUE(bitset1->none()); + auto test = std::make_unique(arr[arr.size() - 1] + 1); + auto bitset2 = creator->index_->NotIn(1, test.get()); + ASSERT_EQ(arr.size(), bitset2->size()); + ASSERT_TRUE(bitset2->any()); +} + +template +inline void +assert_range(const std::unique_ptr>& creator, const std::vector& arr) { + auto test_min = arr[0]; + auto test_max = arr[arr.size() - 1]; + + auto bitset1 = creator->index_->Range(test_min - 1, OperatorType::GT); + ASSERT_EQ(arr.size(), bitset1->size()); + ASSERT_TRUE(bitset1->any()); + + auto bitset2 = creator->index_->Range(test_min, OperatorType::GE); + ASSERT_EQ(arr.size(), bitset2->size()); + ASSERT_TRUE(bitset2->any()); + + auto bitset3 = creator->index_->Range(test_max + 1, OperatorType::LT); + ASSERT_EQ(arr.size(), bitset3->size()); + ASSERT_TRUE(bitset3->any()); + + auto bitset4 = creator->index_->Range(test_max, OperatorType::LE); + ASSERT_EQ(arr.size(), bitset4->size()); + ASSERT_TRUE(bitset4->any()); + + auto bitset5 = creator->index_->Range(test_min, true, test_max, true); + ASSERT_EQ(arr.size(), bitset5->size()); + ASSERT_TRUE(bitset5->any()); +} + +template <> +inline void +assert_in(const std::unique_ptr>& creator, const std::vector& arr) { + auto bitset1 = creator->index_->In(arr.size(), arr.data()); + ASSERT_EQ(arr.size(), bitset1->size()); + ASSERT_TRUE(bitset1->any()); +} + +template <> +inline void +assert_not_in(const std::unique_ptr>& creator, const std::vector& arr) { + auto bitset1 = creator->index_->NotIn(arr.size(), arr.data()); + ASSERT_EQ(arr.size(), bitset1->size()); + ASSERT_TRUE(bitset1->none()); +} + +template <> +inline void +assert_range(const std::unique_ptr>& creator, const std::vector& arr) { + auto test_min = arr[0]; + auto test_max = arr[arr.size() - 1]; + + auto bitset2 = creator->index_->Range(test_min, OperatorType::GE); + ASSERT_EQ(arr.size(), bitset2->size()); + ASSERT_TRUE(bitset2->any()); + + auto bitset4 = creator->index_->Range(test_max, OperatorType::LE); + ASSERT_EQ(arr.size(), bitset4->size()); + ASSERT_TRUE(bitset4->any()); + + auto bitset5 = creator->index_->Range(test_min, true, test_max, true); + ASSERT_EQ(arr.size(), bitset5->size()); + ASSERT_TRUE(bitset5->any()); +} +} // namespace + +template +class TypedScalarIndexTest : public ::testing::Test { + protected: + // void + // SetUp() override { + // } + + // void + // TearDown() override { + // } +}; + +// TODO: it's easy to overflow for int8_t. Design more reasonable ut. +using ArithmeticT = ::testing::Types; + +TYPED_TEST_CASE_P(TypedScalarIndexTest); + +TYPED_TEST_P(TypedScalarIndexTest, Dummy) { + using T = TypeParam; + std::cout << typeid(T()).name() << std::endl; + PrintMapParams(GenParams()); +} + +TYPED_TEST_P(TypedScalarIndexTest, Constructor) { + using T = TypeParam; + for (const auto& tp : GenParams()) { + auto type_params = tp.first; + auto index_params = tp.second; + auto serialized_type_params = generate_type_params(type_params); + auto serialized_index_params = generate_index_params(index_params); + auto creator = + std::make_unique>(serialized_type_params.c_str(), serialized_index_params.c_str()); + } +} + +TYPED_TEST_P(TypedScalarIndexTest, In) { + using T = TypeParam; + for (const auto& tp : GenParams()) { + auto type_params = tp.first; + auto index_params = tp.second; + auto serialized_type_params = generate_type_params(type_params); + auto serialized_index_params = generate_index_params(index_params); + auto creator = + std::make_unique>(serialized_type_params.c_str(), serialized_index_params.c_str()); + auto arr = GenArr(nb); + build_index(creator, arr); + assert_in(creator, arr); + } +} + +TYPED_TEST_P(TypedScalarIndexTest, NotIn) { + using T = TypeParam; + for (const auto& tp : GenParams()) { + auto type_params = tp.first; + auto index_params = tp.second; + auto serialized_type_params = generate_type_params(type_params); + auto serialized_index_params = generate_index_params(index_params); + auto creator = + std::make_unique>(serialized_type_params.c_str(), serialized_index_params.c_str()); + auto arr = GenArr(nb); + build_index(creator, arr); + assert_not_in(creator, arr); + } +} + +TYPED_TEST_P(TypedScalarIndexTest, Range) { + using T = TypeParam; + for (const auto& tp : GenParams()) { + auto type_params = tp.first; + auto index_params = tp.second; + auto serialized_type_params = generate_type_params(type_params); + auto serialized_index_params = generate_index_params(index_params); + auto creator = + std::make_unique>(serialized_type_params.c_str(), serialized_index_params.c_str()); + auto arr = GenArr(nb); + build_index(creator, arr); + assert_range(creator, arr); + } +} + +TYPED_TEST_P(TypedScalarIndexTest, Codec) { + using T = TypeParam; + for (const auto& tp : GenParams()) { + auto type_params = tp.first; + auto index_params = tp.second; + auto serialized_type_params = generate_type_params(type_params); + auto serialized_index_params = generate_index_params(index_params); + auto creator = + std::make_unique>(serialized_type_params.c_str(), serialized_index_params.c_str()); + auto arr = GenArr(nb); + const int64_t dim = 8; // not important here + auto dataset = milvus::knowhere::GenDataset(arr.size(), dim, arr.data()); + creator->Build(dataset); + + auto binary_set = creator->Serialize(); + auto copy_creator = + std::make_unique>(serialized_type_params.c_str(), serialized_index_params.c_str()); + copy_creator->Load(binary_set); + assert_in(copy_creator, arr); + assert_not_in(copy_creator, arr); + assert_range(copy_creator, arr); + } +} + +REGISTER_TYPED_TEST_CASE_P(TypedScalarIndexTest, Dummy, Constructor, In, NotIn, Range, Codec); + +INSTANTIATE_TYPED_TEST_CASE_P(ArithmeticCheck, TypedScalarIndexTest, ArithmeticT); + +class BoolIndexTest : public ::testing::Test { + protected: + void + SetUp() override { + n = 8; + for (size_t i = 0; i < n; i++) { + *(all_true.mutable_data()->Add()) = true; + *(all_false.mutable_data()->Add()) = false; + *(half.mutable_data()->Add()) = (i % 2) == 0; + } + + all_true_ds = GenDsFromPB(all_true); + all_false_ds = GenDsFromPB(all_false); + half_ds = GenDsFromPB(half); + + GenTestParams(); + } + + void + TearDown() override { + delete[](char*)(all_true_ds->Get(milvus::knowhere::meta::TENSOR)); + delete[](char*) all_false_ds->Get(milvus::knowhere::meta::TENSOR); + delete[](char*) half_ds->Get(milvus::knowhere::meta::TENSOR); + } + + private: + void + GenTestParams() { + params = GenBoolParams(); + } + + protected: + schemapb::BoolArray all_true; + schemapb::BoolArray all_false; + schemapb::BoolArray half; + milvus::knowhere::DatasetPtr all_true_ds; + milvus::knowhere::DatasetPtr all_false_ds; + milvus::knowhere::DatasetPtr half_ds; + size_t n; + std::vector params; +}; + +TEST_F(BoolIndexTest, Constructor) { + using T = bool; + for (const auto& tp : params) { + auto type_params = tp.first; + auto index_params = tp.second; + auto serialized_type_params = generate_type_params(type_params); + auto serialized_index_params = generate_index_params(index_params); + auto creator = + std::make_unique>(serialized_type_params.c_str(), serialized_index_params.c_str()); + } +} + +TEST_F(BoolIndexTest, In) { + using T = bool; + for (const auto& tp : params) { + auto type_params = tp.first; + auto index_params = tp.second; + auto serialized_type_params = generate_type_params(type_params); + auto serialized_index_params = generate_index_params(index_params); + + auto true_test = std::make_unique(true); + auto false_test = std::make_unique(false); + + { + auto creator = std::make_unique>(serialized_type_params.c_str(), + serialized_index_params.c_str()); + + creator->Build(all_true_ds); + + auto bitset1 = creator->index_->In(1, true_test.get()); + ASSERT_TRUE(bitset1->any()); + + auto bitset2 = creator->index_->In(1, false_test.get()); + ASSERT_TRUE(bitset2->none()); + } + + { + auto creator = std::make_unique>(serialized_type_params.c_str(), + serialized_index_params.c_str()); + + creator->Build(all_false_ds); + + auto bitset1 = creator->index_->In(1, true_test.get()); + ASSERT_TRUE(bitset1->none()); + + auto bitset2 = creator->index_->In(1, false_test.get()); + ASSERT_TRUE(bitset2->any()); + } + + { + auto creator = std::make_unique>(serialized_type_params.c_str(), + serialized_index_params.c_str()); + + creator->Build(half_ds); + + auto bitset1 = creator->index_->In(1, true_test.get()); + for (size_t i = 0; i < n; i++) { + ASSERT_EQ(bitset1->test(i), (i % 2) == 0); + } + + auto bitset2 = creator->index_->In(1, false_test.get()); + for (size_t i = 0; i < n; i++) { + ASSERT_EQ(bitset2->test(i), (i % 2) != 0); + } + } + } +} + +TEST_F(BoolIndexTest, NotIn) { + using T = bool; + for (const auto& tp : params) { + auto type_params = tp.first; + auto index_params = tp.second; + auto serialized_type_params = generate_type_params(type_params); + auto serialized_index_params = generate_index_params(index_params); + + auto true_test = std::make_unique(true); + auto false_test = std::make_unique(false); + + { + auto creator = std::make_unique>(serialized_type_params.c_str(), + serialized_index_params.c_str()); + + creator->Build(all_true_ds); + + auto bitset1 = creator->index_->NotIn(1, true_test.get()); + ASSERT_TRUE(bitset1->none()); + + auto bitset2 = creator->index_->NotIn(1, false_test.get()); + ASSERT_TRUE(bitset2->any()); + } + + { + auto creator = std::make_unique>(serialized_type_params.c_str(), + serialized_index_params.c_str()); + + creator->Build(all_false_ds); + + auto bitset1 = creator->index_->NotIn(1, true_test.get()); + ASSERT_TRUE(bitset1->any()); + + auto bitset2 = creator->index_->NotIn(1, false_test.get()); + ASSERT_TRUE(bitset2->none()); + } + + { + auto creator = std::make_unique>(serialized_type_params.c_str(), + serialized_index_params.c_str()); + + creator->Build(half_ds); + + auto bitset1 = creator->index_->NotIn(1, true_test.get()); + for (size_t i = 0; i < n; i++) { + ASSERT_EQ(bitset1->test(i), (i % 2) != 0); + } + + auto bitset2 = creator->index_->NotIn(1, false_test.get()); + for (size_t i = 0; i < n; i++) { + ASSERT_EQ(bitset2->test(i), (i % 2) == 0); + } + } + } +} + +TEST_F(BoolIndexTest, Codec) { + using T = bool; + for (const auto& tp : params) { + auto type_params = tp.first; + auto index_params = tp.second; + auto serialized_type_params = generate_type_params(type_params); + auto serialized_index_params = generate_index_params(index_params); + + auto true_test = std::make_unique(true); + auto false_test = std::make_unique(false); + + { + auto creator = std::make_unique>(serialized_type_params.c_str(), + serialized_index_params.c_str()); + + creator->Build(all_true_ds); + + auto copy_creator = std::make_unique>(serialized_type_params.c_str(), + serialized_index_params.c_str()); + copy_creator->Load(creator->Serialize()); + + auto bitset1 = copy_creator->index_->NotIn(1, true_test.get()); + ASSERT_TRUE(bitset1->none()); + + auto bitset2 = copy_creator->index_->NotIn(1, false_test.get()); + ASSERT_TRUE(bitset2->any()); + } + + { + auto creator = std::make_unique>(serialized_type_params.c_str(), + serialized_index_params.c_str()); + + creator->Build(all_false_ds); + + auto copy_creator = std::make_unique>(serialized_type_params.c_str(), + serialized_index_params.c_str()); + copy_creator->Load(creator->Serialize()); + + auto bitset1 = copy_creator->index_->NotIn(1, true_test.get()); + ASSERT_TRUE(bitset1->any()); + + auto bitset2 = copy_creator->index_->NotIn(1, false_test.get()); + ASSERT_TRUE(bitset2->none()); + } + + { + auto creator = std::make_unique>(serialized_type_params.c_str(), + serialized_index_params.c_str()); + + creator->Build(half_ds); + + auto copy_creator = std::make_unique>(serialized_type_params.c_str(), + serialized_index_params.c_str()); + copy_creator->Load(creator->Serialize()); + + auto bitset1 = copy_creator->index_->NotIn(1, true_test.get()); + for (size_t i = 0; i < n; i++) { + ASSERT_EQ(bitset1->test(i), (i % 2) != 0); + } + + auto bitset2 = copy_creator->index_->NotIn(1, false_test.get()); + for (size_t i = 0; i < n; i++) { + ASSERT_EQ(bitset2->test(i), (i % 2) == 0); + } + } + } +} + +class StringIndexTest : public ::testing::Test { + void + SetUp() override { + size_t n = 10; + strs = GenStrArr(n); + *str_arr.mutable_data() = {strs.begin(), strs.end()}; + str_ds = GenDsFromPB(str_arr); + + GenTestParams(); + } + + void + TearDown() override { + delete[](char*)(str_ds->Get(milvus::knowhere::meta::TENSOR)); + } + + private: + void + GenTestParams() { + params = GenStringParams(); + } + + protected: + std::vector strs; + schemapb::StringArray str_arr; + milvus::knowhere::DatasetPtr str_ds; + std::vector params; +}; + +TEST_F(StringIndexTest, Constructor) { + using T = std::string; + for (const auto& tp : params) { + auto type_params = tp.first; + auto index_params = tp.second; + auto serialized_type_params = generate_type_params(type_params); + auto serialized_index_params = generate_index_params(index_params); + auto creator = + std::make_unique>(serialized_type_params.c_str(), serialized_index_params.c_str()); + } +} + +TEST_F(StringIndexTest, In) { + using T = std::string; + for (const auto& tp : params) { + PrintMapParam(tp); + + auto type_params = tp.first; + auto index_params = tp.second; + + auto serialized_type_params = generate_type_params(type_params); + auto serialized_index_params = generate_index_params(index_params); + + auto creator = + std::make_unique>(serialized_type_params.c_str(), serialized_index_params.c_str()); + creator->Build(str_ds); + assert_in(creator, strs); + } +} + +TEST_F(StringIndexTest, NotIn) { + using T = std::string; + for (const auto& tp : params) { + auto type_params = tp.first; + auto index_params = tp.second; + auto serialized_type_params = generate_type_params(type_params); + auto serialized_index_params = generate_index_params(index_params); + + auto creator = + std::make_unique>(serialized_type_params.c_str(), serialized_index_params.c_str()); + creator->Build(str_ds); + assert_not_in(creator, strs); + } +} + +TEST_F(StringIndexTest, Range) { + using T = std::string; + for (const auto& tp : params) { + auto type_params = tp.first; + auto index_params = tp.second; + auto serialized_type_params = generate_type_params(type_params); + auto serialized_index_params = generate_index_params(index_params); + + auto creator = + std::make_unique>(serialized_type_params.c_str(), serialized_index_params.c_str()); + creator->Build(str_ds); + assert_range(creator, strs); + } +} + +TEST_F(StringIndexTest, Codec) { + using T = std::string; + for (const auto& tp : params) { + auto type_params = tp.first; + auto index_params = tp.second; + auto serialized_type_params = generate_type_params(type_params); + auto serialized_index_params = generate_index_params(index_params); + + auto creator = + std::make_unique>(serialized_type_params.c_str(), serialized_index_params.c_str()); + creator->Build(str_ds); + + auto copy_creator = + std::make_unique>(serialized_type_params.c_str(), serialized_index_params.c_str()); + auto binary_set = creator->Serialize(); + copy_creator->Load(binary_set); + assert_in(copy_creator, strs); + assert_not_in(copy_creator, strs); + assert_range(copy_creator, strs); + } +} diff --git a/internal/core/unittest/test_segcore.cpp b/internal/core/unittest/test_segcore.cpp index ea668c42197f2c70f49cfaa85380c2ab5faff0ab..3fec7cbef4368f046aad19e2e52b53062c2e5df8 100644 --- a/internal/core/unittest/test_segcore.cpp +++ b/internal/core/unittest/test_segcore.cpp @@ -98,76 +98,76 @@ TEST(SegmentCoreTest, MockTest) { // Test insert column-based data TEST(SegmentCoreTest, MockTest2) { - using namespace milvus::segcore; - using namespace milvus::engine; - - // schema - auto schema = std::make_shared(); - schema->AddDebugField("fakevec", DataType::VECTOR_FLOAT, 16, MetricType::METRIC_L2); - schema->AddDebugField("age", DataType::INT32); - - // generate random row-based data - std::vector row_data; - std::vector timestamps; - std::vector uids; - int N = 10000; // number of records - std::default_random_engine e(67); - for (int i = 0; i < N; ++i) { - uids.push_back(100000 + i); - timestamps.push_back(0); - // append vec - float vec[16]; - for (auto& x : vec) { - x = e() % 2000 * 0.001 - 1.0; - } - row_data.insert(row_data.end(), (const char*)std::begin(vec), (const char*)std::end(vec)); - int age = e() % 100; - row_data.insert(row_data.end(), (const char*)&age, ((const char*)&age) + sizeof(age)); - } - auto line_sizeof = (sizeof(int) + sizeof(float) * 16); - assert(row_data.size() == line_sizeof * N); - - int64_t size = N; - const int64_t* uids_raw = uids.data(); - const Timestamp* timestamps_raw = timestamps.data(); - std::vector> ordering(size); // timestamp, pk, order_index - for (int i = 0; i < size; ++i) { - ordering[i] = std::make_tuple(timestamps_raw[i], uids_raw[i], i); - } - std::sort(ordering.begin(), ordering.end()); // sort according to timestamp - - // convert row-based data to column-based data accordingly - auto sizeof_infos = schema->get_sizeof_infos(); - std::vector offset_infos(schema->size() + 1, 0); - std::partial_sum(sizeof_infos.begin(), sizeof_infos.end(), offset_infos.begin() + 1); - std::vector> entities(schema->size()); - - for (int fid = 0; fid < schema->size(); ++fid) { - auto len = sizeof_infos[fid]; - entities[fid].resize(len * size); - } - - auto raw_data = row_data.data(); - std::vector sorted_uids(size); - std::vector sorted_timestamps(size); - for (int index = 0; index < size; ++index) { - auto [t, uid, order_index] = ordering[index]; - sorted_timestamps[index] = t; - sorted_uids[index] = uid; - for (int fid = 0; fid < schema->size(); ++fid) { - auto len = sizeof_infos[fid]; - auto offset = offset_infos[fid]; - auto src = raw_data + order_index * line_sizeof + offset; - auto dst = entities[fid].data() + index * len; - memcpy(dst, src, len); - } - } - - // insert column-based data - ColumnBasedRawData data_chunk{entities, N}; - auto segment = CreateGrowingSegment(schema); - auto reserved_begin = segment->PreInsert(N); - segment->Insert(reserved_begin, size, sorted_uids.data(), sorted_timestamps.data(), data_chunk); + using namespace milvus::segcore; + using namespace milvus::engine; + + // schema + auto schema = std::make_shared(); + schema->AddDebugField("fakevec", DataType::VECTOR_FLOAT, 16, MetricType::METRIC_L2); + schema->AddDebugField("age", DataType::INT32); + + // generate random row-based data + std::vector row_data; + std::vector timestamps; + std::vector uids; + int N = 10000; // number of records + std::default_random_engine e(67); + for (int i = 0; i < N; ++i) { + uids.push_back(100000 + i); + timestamps.push_back(0); + // append vec + float vec[16]; + for (auto& x : vec) { + x = e() % 2000 * 0.001 - 1.0; + } + row_data.insert(row_data.end(), (const char*)std::begin(vec), (const char*)std::end(vec)); + int age = e() % 100; + row_data.insert(row_data.end(), (const char*)&age, ((const char*)&age) + sizeof(age)); + } + auto line_sizeof = (sizeof(int) + sizeof(float) * 16); + assert(row_data.size() == line_sizeof * N); + + int64_t size = N; + const int64_t* uids_raw = uids.data(); + const Timestamp* timestamps_raw = timestamps.data(); + std::vector> ordering(size); // timestamp, pk, order_index + for (int i = 0; i < size; ++i) { + ordering[i] = std::make_tuple(timestamps_raw[i], uids_raw[i], i); + } + std::sort(ordering.begin(), ordering.end()); // sort according to timestamp + + // convert row-based data to column-based data accordingly + auto sizeof_infos = schema->get_sizeof_infos(); + std::vector offset_infos(schema->size() + 1, 0); + std::partial_sum(sizeof_infos.begin(), sizeof_infos.end(), offset_infos.begin() + 1); + std::vector> entities(schema->size()); + + for (int fid = 0; fid < schema->size(); ++fid) { + auto len = sizeof_infos[fid]; + entities[fid].resize(len * size); + } + + auto raw_data = row_data.data(); + std::vector sorted_uids(size); + std::vector sorted_timestamps(size); + for (int index = 0; index < size; ++index) { + auto [t, uid, order_index] = ordering[index]; + sorted_timestamps[index] = t; + sorted_uids[index] = uid; + for (int fid = 0; fid < schema->size(); ++fid) { + auto len = sizeof_infos[fid]; + auto offset = offset_infos[fid]; + auto src = raw_data + order_index * line_sizeof + offset; + auto dst = entities[fid].data() + index * len; + memcpy(dst, src, len); + } + } + + // insert column-based data + ColumnBasedRawData data_chunk{entities, N}; + auto segment = CreateGrowingSegment(schema); + auto reserved_begin = segment->PreInsert(N); + segment->Insert(reserved_begin, size, sorted_uids.data(), sorted_timestamps.data(), data_chunk); } TEST(SegmentCoreTest, SmallIndex) { diff --git a/internal/core/unittest/test_utils/indexbuilder_test_utils.h b/internal/core/unittest/test_utils/indexbuilder_test_utils.h index ee218b353bd54c44ce935cd2dc6eb9bd46c11a32..ad3ea7f3aee14cd6b8424052a6e4423c5f2b195f 100644 --- a/internal/core/unittest/test_utils/indexbuilder_test_utils.h +++ b/internal/core/unittest/test_utils/indexbuilder_test_utils.h @@ -22,18 +22,29 @@ #include #include "pb/index_cgo_msg.pb.h" -#include "indexbuilder/IndexWrapper.h" + +#include "indexbuilder/VecIndexCreator.h" #include "indexbuilder/index_c.h" #include "DataGen.h" #include "indexbuilder/utils.h" +#include "indexbuilder/helper.h" +#define private public +#include "indexbuilder/ScalarIndexCreator.h" -constexpr int64_t DIM = 128; +constexpr int64_t DIM = 8; constexpr int64_t NQ = 10; constexpr int64_t K = 4; #ifdef MILVUS_GPU_VERSION int DEVICEID = 0; #endif +namespace indexcgo = milvus::proto::indexcgo; +namespace schemapb = milvus::proto::schema; +using milvus::indexbuilder::MapParams; +using milvus::indexbuilder::ScalarIndexCreator; +using milvus::knowhere::scalar::OperatorType; +using ScalarTestParams = std::pair; + namespace { auto generate_conf(const milvus::knowhere::IndexType& index_type, const milvus::knowhere::MetricType& metric_type) { @@ -234,7 +245,7 @@ GenDataset(int64_t N, const milvus::knowhere::MetricType& metric_type, bool is_b } } -using QueryResultPtr = std::unique_ptr; +using QueryResultPtr = std::unique_ptr; void PrintQueryResult(const QueryResultPtr& result) { auto nq = result->nq; @@ -327,4 +338,116 @@ CheckDistances(const QueryResultPtr& result, } } } + +auto +generate_type_params(const MapParams& m) { + indexcgo::TypeParams p; + for (const auto& [k, v] : m) { + auto kv = p.add_params(); + kv->set_key(k); + kv->set_value(v); + } + std::string str; + auto ok = google::protobuf::TextFormat::PrintToString(p, &str); + Assert(ok); + return str; +} + +auto +generate_index_params(const MapParams& m) { + indexcgo::IndexParams p; + for (const auto& [k, v] : m) { + auto kv = p.add_params(); + kv->set_key(k); + kv->set_value(v); + } + std::string str; + auto ok = google::protobuf::TextFormat::PrintToString(p, &str); + Assert(ok); + return str; +} + +// TODO: std::is_arithmetic_v, hard to compare float point value. std::is_integral_v. +template >> +inline auto +GenArr(int64_t n) { + auto max_i8 = std::numeric_limits::max() - 1; + std::vector arr; + arr.resize(n); + for (int64_t i = 0; i < n; i++) { + arr[i] = static_cast(rand() % max_i8); + } + std::sort(arr.begin(), arr.end()); + return arr; +} + +inline auto +GenStrArr(int64_t n) { + using T = std::string; + std::vector arr; + arr.resize(n); + for (int64_t i = 0; i < n; i++) { + auto gen = std::to_string(std::rand()); + arr[i] = gen; + } + std::sort(arr.begin(), arr.end()); + return arr; +} + +template >> +inline std::vector +GenParams() { + std::vector ret; + ret.emplace_back(ScalarTestParams(MapParams(), {{"index_type", "inverted_index"}})); + ret.emplace_back(ScalarTestParams(MapParams(), {{"index_type", "flat"}})); + return ret; +} + +std::vector +GenBoolParams() { + std::vector ret; + ret.emplace_back(ScalarTestParams(MapParams(), {{"index_type", "inverted_index"}})); + ret.emplace_back(ScalarTestParams(MapParams(), {{"index_type", "flat"}})); + return ret; +} + +std::vector +GenStringParams() { + std::vector ret; + ret.emplace_back(ScalarTestParams(MapParams(), {{"index_type", "marisa-trie"}})); + return ret; +} + +void +PrintMapParam(const ScalarTestParams& tp) { + for (const auto& [k, v] : tp.first) { + std::cout << "k: " << k << ", v: " << v << std::endl; + } + for (const auto& [k, v] : tp.second) { + std::cout << "k: " << k << ", v: " << v << std::endl; + } +} + +void +PrintMapParams(const std::vector& tps) { + for (const auto& tp : tps) { + PrintMapParam(tp); + } +} + +template +inline void +build_index(const std::unique_ptr>& creator, const std::vector& arr) { + const int64_t dim = 8; // not important here + auto dataset = milvus::knowhere::GenDataset(arr.size(), dim, arr.data()); + creator->Build(dataset); +} + +// memory generated by this function should be freed by the caller. +auto +GenDsFromPB(const google::protobuf::Message& msg) { + auto data = new char[msg.ByteSize()]; + msg.SerializeToArray(data, msg.ByteSize()); + return milvus::knowhere::GenDataset(msg.ByteSize(), 8, data); +} } // namespace diff --git a/internal/indexcoord/util.go b/internal/indexcoord/util.go index ccd89a144be9c63779f4b58c5daf02e296152ad5..f230cf7ebcfd4ea03845cbb136fd53992a26dfac 100644 --- a/internal/indexcoord/util.go +++ b/internal/indexcoord/util.go @@ -54,7 +54,10 @@ func estimateIndexSize(dim int64, numRows int64, dataType schemapb.DataType) (ui return uint64(dim) / 8 * uint64(numRows), nil } - errMsg := "the field to build index must be a vector field" - log.Error(errMsg) - return 0, errors.New(errMsg) + // TODO: optimize here. + return 0, nil + + // errMsg := "the field to build index must be a vector field" + // log.Error(errMsg) + // return 0, errors.New(errMsg) } diff --git a/internal/indexcoord/util_test.go b/internal/indexcoord/util_test.go index b5e297283fb58f72998f388e9d58525358311ba1..c70ca1e4d0c1b33b067246f24150ccc13c037961 100644 --- a/internal/indexcoord/util_test.go +++ b/internal/indexcoord/util_test.go @@ -79,6 +79,8 @@ func Test_estimateIndexSize(t *testing.T) { assert.Equal(t, uint64(200), memorySize) memorySize, err = estimateIndexSize(10, 100, schemapb.DataType_Float) - assert.Error(t, err) + assert.Nil(t, err) assert.Equal(t, uint64(0), memorySize) + // assert.Error(t, err) + // assert.Equal(t, uint64(0), memorySize) } diff --git a/internal/indexnode/index.go b/internal/indexnode/index.go deleted file mode 100644 index 6fbb2147e74fd90fed11769d23ddaa816faa6601..0000000000000000000000000000000000000000 --- a/internal/indexnode/index.go +++ /dev/null @@ -1,315 +0,0 @@ -// Licensed to the LF AI & Data foundation under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package indexnode - -/* - -#cgo CFLAGS: -I${SRCDIR}/../core/output/include - -#cgo darwin LDFLAGS: -L${SRCDIR}/../core/output/lib -lmilvus_common -lmilvus_indexbuilder -Wl,-rpath,"${SRCDIR}/../core/output/lib" -#cgo linux LDFLAGS: -L${SRCDIR}/../core/output/lib -lmilvus_common -lmilvus_indexbuilder -Wl,-rpath=${SRCDIR}/../core/output/lib -#cgo windows LDFLAGS: -L${SRCDIR}/../core/output/lib -lmilvus_common -lmilvus_indexbuilder -Wl,-rpath=${SRCDIR}/../core/output/lib - -#include // free -#include "segcore/collection_c.h" -#include "indexbuilder/index_c.h" -#include "common/vector_index_c.h" - -*/ -import "C" - -import ( - "errors" - "fmt" - "path/filepath" - "runtime" - "unsafe" - - "go.uber.org/zap" - - "github.com/golang/protobuf/proto" - "github.com/milvus-io/milvus/internal/log" - "github.com/milvus-io/milvus/internal/proto/commonpb" - "github.com/milvus-io/milvus/internal/proto/indexcgopb" - "github.com/milvus-io/milvus/internal/storage" -) - -// Blob is an alias for the storage.Blob type -type Blob = storage.Blob - -// Index is an interface used to call the interface to build the index task in 'C'. -type Index interface { - Serialize() ([]*Blob, error) - Load([]*Blob) error - BuildFloatVecIndexWithoutIds(vectors []float32) error - BuildBinaryVecIndexWithoutIds(vectors []byte) error - Delete() error -} - -// CIndex is a pointer used to access 'CGO'. -type CIndex struct { - indexPtr C.CIndex - close bool -} - -func GetBinarySetKeys(cBinarySet C.CBinarySet) ([]string, error) { - size := int(C.GetBinarySetSize(cBinarySet)) - if size == 0 { - return nil, fmt.Errorf("BinarySet size is zero!") - } - datas := make([]unsafe.Pointer, size) - - C.GetBinarySetKeys(cBinarySet, unsafe.Pointer(&datas[0])) - ret := make([]string, size) - for i := 0; i < size; i++ { - ret[i] = C.GoString((*C.char)(datas[i])) - } - - return ret, nil -} - -func GetBinarySetValue(cBinarySet C.CBinarySet, key string) ([]byte, error) { - cIndexKey := C.CString(key) - defer C.free(unsafe.Pointer(cIndexKey)) - ret := C.GetBinarySetValueSize(cBinarySet, cIndexKey) - size := int(ret) - if size == 0 { - return nil, fmt.Errorf("GetBinarySetValueSize size is zero!") - } - value := make([]byte, size) - status := C.CopyBinarySetValue(unsafe.Pointer(&value[0]), cIndexKey, cBinarySet) - - if err := HandleCStatus(&status, "CopyBinarySetValue failed"); err != nil { - return nil, err - } - - return value, nil -} - -func (index *CIndex) Serialize() ([]*Blob, error) { - var cBinarySet C.CBinarySet - - status := C.SerializeToBinarySet(index.indexPtr, &cBinarySet) - defer func() { - if cBinarySet != nil { - C.DeleteBinarySet(cBinarySet) - } - }() - if err := HandleCStatus(&status, "SerializeToBinarySet failed"); err != nil { - return nil, err - } - - keys, err := GetBinarySetKeys(cBinarySet) - if err != nil { - return nil, err - } - ret := make([]*Blob, 0) - for _, key := range keys { - value, err := GetBinarySetValue(cBinarySet, key) - if err != nil { - return nil, err - } - blob := &Blob{ - Key: key, - Value: value, - } - ret = append(ret, blob) - } - - return ret, nil -} - -// Serialize serializes vector data into bytes data so that it can be accessed in 'C'. -/* -func (index *CIndex) Serialize() ([]*Blob, error) { - var cBinary C.CBinary - - status := C.SerializeToSlicedBuffer(index.indexPtr, &cBinary) - defer func() { - if cBinary != nil { - C.DeleteCBinary(cBinary) - } - }() - if err := HandleCStatus(&status, "SerializeToSlicedBuffer failed"); err != nil { - return nil, err - } - - binarySize := C.GetCBinarySize(cBinary) - binaryData := make([]byte, binarySize) - C.GetCBinaryData(cBinary, unsafe.Pointer(&binaryData[0])) - - var blobs indexcgopb.BinarySet - err := proto.Unmarshal(binaryData, &blobs) - if err != nil { - return nil, err - } - - ret := make([]*Blob, 0) - for _, data := range blobs.Datas { - ret = append(ret, &Blob{Key: data.Key, Value: data.Value}) - } - - return ret, nil -} -*/ - -func (index *CIndex) Load(blobs []*Blob) error { - var cBinarySet C.CBinarySet - status := C.NewBinarySet(&cBinarySet) - defer C.DeleteBinarySet(cBinarySet) - - if err := HandleCStatus(&status, "CIndex Load2 NewBinarySet failed"); err != nil { - return err - } - for _, blob := range blobs { - key := blob.Key - byteIndex := blob.Value - indexPtr := unsafe.Pointer(&byteIndex[0]) - indexLen := C.int64_t(len(byteIndex)) - binarySetKey := filepath.Base(key) - log.Debug("", zap.String("index key", binarySetKey)) - indexKey := C.CString(binarySetKey) - status = C.AppendIndexBinary(cBinarySet, indexPtr, indexLen, indexKey) - C.free(unsafe.Pointer(indexKey)) - if err := HandleCStatus(&status, "CIndex Load AppendIndexBinary failed"); err != nil { - return err - } - } - status = C.LoadFromBinarySet(index.indexPtr, cBinarySet) - return HandleCStatus(&status, "AppendIndex failed") -} - -// Load loads data from 'C'. -/* -func (index *CIndex) Load(blobs []*Blob) error { - binarySet := &indexcgopb.BinarySet{Datas: make([]*indexcgopb.Binary, 0)} - for _, blob := range blobs { - binarySet.Datas = append(binarySet.Datas, &indexcgopb.Binary{Key: blob.Key, Value: blob.Value}) - } - - datas, err2 := proto.Marshal(binarySet) - if err2 != nil { - return err2 - } - - status := C.LoadFromSlicedBuffer(index.indexPtr, (*C.char)(unsafe.Pointer(&datas[0])), (C.int32_t)(len(datas))) - return HandleCStatus(&status, "LoadFromSlicedBuffer failed") -} -*/ - -// BuildFloatVecIndexWithoutIds builds indexes for float vector. -func (index *CIndex) BuildFloatVecIndexWithoutIds(vectors []float32) error { - /* - CStatus - BuildFloatVecIndexWithoutIds(CIndex index, int64_t float_value_num, const float* vectors); - */ - log.Debug("before BuildFloatVecIndexWithoutIds") - status := C.BuildFloatVecIndexWithoutIds(index.indexPtr, (C.int64_t)(len(vectors)), (*C.float)(&vectors[0])) - return HandleCStatus(&status, "BuildFloatVecIndexWithoutIds failed") -} - -// BuildBinaryVecIndexWithoutIds builds indexes for binary vector. -func (index *CIndex) BuildBinaryVecIndexWithoutIds(vectors []byte) error { - /* - CStatus - BuildBinaryVecIndexWithoutIds(CIndex index, int64_t data_size, const uint8_t* vectors); - */ - status := C.BuildBinaryVecIndexWithoutIds(index.indexPtr, (C.int64_t)(len(vectors)), (*C.uint8_t)(&vectors[0])) - return HandleCStatus(&status, "BuildBinaryVecIndexWithoutIds failed") -} - -// Delete removes the pointer to build the index in 'C'. we can ensure that it is idempotent. -func (index *CIndex) Delete() error { - /* - void - DeleteIndex(CIndex index); - */ - if index.close { - return nil - } - C.DeleteIndex(index.indexPtr) - index.close = true - return nil -} - -// NewCIndex creates a new pointer to build the index in 'C'. -func NewCIndex(typeParams, indexParams map[string]string) (Index, error) { - protoTypeParams := &indexcgopb.TypeParams{ - Params: make([]*commonpb.KeyValuePair, 0), - } - for key, value := range typeParams { - protoTypeParams.Params = append(protoTypeParams.Params, &commonpb.KeyValuePair{Key: key, Value: value}) - } - typeParamsStr := proto.MarshalTextString(protoTypeParams) - - protoIndexParams := &indexcgopb.IndexParams{ - Params: make([]*commonpb.KeyValuePair, 0), - } - for key, value := range indexParams { - protoIndexParams.Params = append(protoIndexParams.Params, &commonpb.KeyValuePair{Key: key, Value: value}) - } - indexParamsStr := proto.MarshalTextString(protoIndexParams) - - typeParamsPointer := C.CString(typeParamsStr) - indexParamsPointer := C.CString(indexParamsStr) - defer C.free(unsafe.Pointer(typeParamsPointer)) - defer C.free(unsafe.Pointer(indexParamsPointer)) - - /* - CStatus - CreateIndex(const char* serialized_type_params, - const char* serialized_index_params, - CIndex* res_index); - */ - var indexPtr C.CIndex - log.Debug("Start to create index ...", zap.String("params", indexParamsStr)) - status := C.CreateIndex(typeParamsPointer, indexParamsPointer, &indexPtr) - if err := HandleCStatus(&status, "CreateIndex failed"); err != nil { - return nil, err - } - log.Debug("Successfully create index ...") - - index := &CIndex{ - indexPtr: indexPtr, - close: false, - } - runtime.SetFinalizer(index, func(index *CIndex) { - if index != nil && !index.close { - log.Error("there is leakage in index object, please check.") - } - }) - return index, nil -} - -// HandleCStatus deal with the error returned from CGO -func HandleCStatus(status *C.CStatus, extraInfo string) error { - if status.error_code == 0 { - return nil - } - errorCode := status.error_code - errorName, ok := commonpb.ErrorCode_name[int32(errorCode)] - if !ok { - errorName = "UnknownError" - } - errorMsg := C.GoString(status.error_msg) - defer C.free(unsafe.Pointer(status.error_msg)) - - finalMsg := fmt.Sprintf("[%s] %s", errorName, errorMsg) - logMsg := fmt.Sprintf("%s, C Runtime Exception: %s\n", extraInfo, finalMsg) - log.Warn(logMsg) - return errors.New(finalMsg) -} diff --git a/internal/indexnode/index_test.go b/internal/indexnode/index_test.go index e518108c6a55bd90050854d150b4f4351031e947..235d00a908a3674c9a1744029b0168988d621371 100644 --- a/internal/indexnode/index_test.go +++ b/internal/indexnode/index_test.go @@ -1,216 +1,13 @@ -// Licensed to the LF AI & Data foundation under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build linux -// +build linux - package indexnode -import ( - "fmt" - "math/rand" - "strconv" - "testing" - - "github.com/stretchr/testify/assert" -) +import "math/rand" const ( - // index type - IndexFaissIDMap = "FLAT" - IndexFaissIVFFlat = "IVF_FLAT" - IndexFaissIVFPQ = "IVF_PQ" - IndexFaissIVFSQ8 = "IVF_SQ8" - IndexFaissIVFSQ8H = "IVF_SQ8_HYBRID" - IndexFaissBinIDMap = "BIN_FLAT" - IndexFaissBinIVFFlat = "BIN_IVF_FLAT" - IndexNsg = "NSG" - - IndexHNSW = "HNSW" - IndexRHNSWFlat = "RHNSW_FLAT" - IndexRHNSWPQ = "RHNSW_PQ" - IndexRHNSWSQ = "RHNSW_SQ" - IndexANNOY = "ANNOY" - IndexNGTPANNG = "NGT_PANNG" - IndexNGTONNG = "NGT_ONNG" - - // metric type - L2 = "L2" - IP = "IP" - hamming = "HAMMING" - Jaccard = "JACCARD" - tanimoto = "TANIMOTO" - - dim = 8 - nlist = 100 - m = 4 - nbits = 8 - nb = 10000 - nprobe = 8 - sliceSize = 4 - efConstruction = 200 - ef = 200 - edgeSize = 10 - epsilon = 0.1 - maxSearchEdges = 50 + dim = 8 + nb = 10000 + nprobe = 8 ) -type testCase struct { - indexType string - metricType string - isBinary bool -} - -func generateFloatVectorTestCases() []testCase { - return []testCase{ - {IndexFaissIDMap, L2, false}, - {IndexFaissIDMap, IP, false}, - {IndexFaissIVFFlat, L2, false}, - {IndexFaissIVFFlat, IP, false}, - {IndexFaissIVFPQ, L2, false}, - {IndexFaissIVFPQ, IP, false}, - {IndexFaissIVFSQ8, L2, false}, - {IndexFaissIVFSQ8, IP, false}, - //{IndexFaissIVFSQ8H, L2, false}, // TODO: enable gpu - //{IndexFaissIVFSQ8H, IP, false}, - {IndexNsg, L2, false}, - {IndexNsg, IP, false}, - //{IndexHNSW, L2, false}, // TODO: fix json parse exception - //{IndexHNSW, IP, false}, - //{IndexRHNSWFlat, L2, false}, - //{IndexRHNSWFlat, IP, false}, - //{IndexRHNSWPQ, L2, false}, - //{IndexRHNSWPQ, IP, false}, - //{IndexRHNSWSQ, L2, false}, - //{IndexRHNSWSQ, IP, false}, - {IndexANNOY, L2, false}, - {IndexANNOY, IP, false}, - {IndexNGTPANNG, L2, false}, - {IndexNGTPANNG, IP, false}, - {IndexNGTONNG, L2, false}, - {IndexNGTONNG, IP, false}, - } -} - -func generateBinaryVectorTestCases() []testCase { - return []testCase{ - {IndexFaissBinIVFFlat, Jaccard, true}, - {IndexFaissBinIVFFlat, hamming, true}, - {IndexFaissBinIVFFlat, tanimoto, true}, - {IndexFaissBinIDMap, Jaccard, true}, - {IndexFaissBinIDMap, hamming, true}, - } -} - -func generateTestCases() []testCase { - return append(generateFloatVectorTestCases(), generateBinaryVectorTestCases()...) -} - -func generateParams(indexType, metricType string) (map[string]string, map[string]string) { - typeParams := make(map[string]string) - indexParams := make(map[string]string) - indexParams["index_type"] = indexType - indexParams["metric_type"] = metricType - if indexType == IndexFaissIDMap { // float vector - indexParams["dim"] = strconv.Itoa(dim) - indexParams["SLICE_SIZE"] = strconv.Itoa(sliceSize) - } else if indexType == IndexFaissIVFFlat { - indexParams["dim"] = strconv.Itoa(dim) - indexParams["nlist"] = strconv.Itoa(nlist) - } else if indexType == IndexFaissIVFPQ { - indexParams["dim"] = strconv.Itoa(dim) - indexParams["nlist"] = strconv.Itoa(nlist) - indexParams["m"] = strconv.Itoa(m) - indexParams["nbits"] = strconv.Itoa(nbits) - indexParams["SLICE_SIZE"] = strconv.Itoa(sliceSize) - } else if indexType == IndexFaissIVFSQ8 { - indexParams["dim"] = strconv.Itoa(dim) - indexParams["nlist"] = strconv.Itoa(nlist) - indexParams["nbits"] = strconv.Itoa(nbits) - indexParams["SLICE_SIZE"] = strconv.Itoa(sliceSize) - } else if indexType == IndexFaissIVFSQ8H { - // TODO: enable gpu - } else if indexType == IndexNsg { - indexParams["dim"] = strconv.Itoa(dim) - indexParams["nlist"] = strconv.Itoa(163) - indexParams["nprobe"] = strconv.Itoa(nprobe) - indexParams["knng"] = strconv.Itoa(20) - indexParams["search_length"] = strconv.Itoa(40) - indexParams["out_degree"] = strconv.Itoa(30) - indexParams["candidate_pool_size"] = strconv.Itoa(100) - } else if indexType == IndexHNSW { - indexParams["dim"] = strconv.Itoa(dim) - indexParams["m"] = strconv.Itoa(16) - indexParams["efConstruction"] = strconv.Itoa(efConstruction) - indexParams["ef"] = strconv.Itoa(ef) - } else if indexType == IndexRHNSWFlat { - indexParams["dim"] = strconv.Itoa(dim) - indexParams["m"] = strconv.Itoa(16) - indexParams["efConstruction"] = strconv.Itoa(efConstruction) - indexParams["ef"] = strconv.Itoa(ef) - indexParams["SLICE_SIZE"] = strconv.Itoa(sliceSize) - } else if indexType == IndexRHNSWPQ { - indexParams["dim"] = strconv.Itoa(dim) - indexParams["m"] = strconv.Itoa(16) - indexParams["efConstruction"] = strconv.Itoa(efConstruction) - indexParams["ef"] = strconv.Itoa(ef) - indexParams["SLICE_SIZE"] = strconv.Itoa(sliceSize) - indexParams["PQM"] = strconv.Itoa(8) - } else if indexType == IndexRHNSWSQ { - indexParams["dim"] = strconv.Itoa(dim) - indexParams["m"] = strconv.Itoa(16) - indexParams["efConstruction"] = strconv.Itoa(efConstruction) - indexParams["ef"] = strconv.Itoa(ef) - indexParams["SLICE_SIZE"] = strconv.Itoa(sliceSize) - } else if indexType == IndexANNOY { - indexParams["dim"] = strconv.Itoa(dim) - indexParams["n_trees"] = strconv.Itoa(4) - indexParams["search_k"] = strconv.Itoa(100) - indexParams["SLICE_SIZE"] = strconv.Itoa(sliceSize) - } else if indexType == IndexNGTPANNG { - indexParams["dim"] = strconv.Itoa(dim) - indexParams["edge_size"] = strconv.Itoa(edgeSize) - indexParams["epsilon"] = fmt.Sprint(epsilon) - indexParams["max_search_edges"] = strconv.Itoa(maxSearchEdges) - indexParams["forcedly_pruned_edge_size"] = strconv.Itoa(60) - indexParams["selectively_pruned_edge_size"] = strconv.Itoa(30) - indexParams["SLICE_SIZE"] = strconv.Itoa(sliceSize) - } else if indexType == IndexNGTONNG { - indexParams["dim"] = strconv.Itoa(dim) - indexParams["edge_size"] = strconv.Itoa(edgeSize) - indexParams["epsilon"] = fmt.Sprint(epsilon) - indexParams["max_search_edges"] = strconv.Itoa(maxSearchEdges) - indexParams["outgoing_edge_size"] = strconv.Itoa(5) - indexParams["incoming_edge_size"] = strconv.Itoa(40) - indexParams["SLICE_SIZE"] = strconv.Itoa(sliceSize) - } else if indexType == IndexFaissBinIVFFlat { // binary vector - indexParams["dim"] = strconv.Itoa(dim) - indexParams["nlist"] = strconv.Itoa(nlist) - indexParams["m"] = strconv.Itoa(m) - indexParams["nbits"] = strconv.Itoa(nbits) - indexParams["SLICE_SIZE"] = strconv.Itoa(sliceSize) - } else if indexType == IndexFaissBinIDMap { - indexParams["dim"] = strconv.Itoa(dim) - } else { - panic("") - } - - return typeParams, indexParams -} - func generateFloatVectors() []float32 { vectors := make([]float32, 0) for i := 0; i < nb; i++ { @@ -230,134 +27,3 @@ func generateBinaryVectors() []byte { } return vectors } - -func TestCIndex_New(t *testing.T) { - for _, c := range generateTestCases() { - typeParams, indexParams := generateParams(c.indexType, c.metricType) - - index, err := NewCIndex(typeParams, indexParams) - assert.Equal(t, err, nil) - assert.NotEqual(t, index, nil) - - err = index.Delete() - assert.Equal(t, err, nil) - } -} - -func TestCIndex_BuildFloatVecIndexWithoutIds(t *testing.T) { - for _, c := range generateFloatVectorTestCases() { - typeParams, indexParams := generateParams(c.indexType, c.metricType) - - index, err := NewCIndex(typeParams, indexParams) - assert.Equal(t, err, nil) - assert.NotEqual(t, index, nil) - - vectors := generateFloatVectors() - err = index.BuildFloatVecIndexWithoutIds(vectors) - assert.Equal(t, err, nil) - - err = index.Delete() - assert.Equal(t, err, nil) - } -} - -func TestCIndex_BuildBinaryVecIndexWithoutIds(t *testing.T) { - for _, c := range generateBinaryVectorTestCases() { - typeParams, indexParams := generateParams(c.indexType, c.metricType) - - index, err := NewCIndex(typeParams, indexParams) - assert.Equal(t, err, nil) - assert.NotEqual(t, index, nil) - - vectors := generateBinaryVectors() - err = index.BuildBinaryVecIndexWithoutIds(vectors) - assert.Equal(t, err, nil) - - err = index.Delete() - assert.Equal(t, err, nil) - } -} - -func TestCIndex_Codec(t *testing.T) { - for _, c := range generateTestCases() { - typeParams, indexParams := generateParams(c.indexType, c.metricType) - - index, err := NewCIndex(typeParams, indexParams) - assert.Equal(t, err, nil) - assert.NotEqual(t, index, nil) - - if !c.isBinary { - vectors := generateFloatVectors() - err = index.BuildFloatVecIndexWithoutIds(vectors) - assert.Equal(t, err, nil) - } else { - vectors := generateBinaryVectors() - err = index.BuildBinaryVecIndexWithoutIds(vectors) - assert.Equal(t, err, nil) - } - - blobs, err := index.Serialize() - assert.Equal(t, err, nil) - - copyIndex, err := NewCIndex(typeParams, indexParams) - assert.NotEqual(t, copyIndex, nil) - assert.Equal(t, err, nil) - err = copyIndex.Load(blobs) - assert.Equal(t, err, nil) - copyBlobs, err := copyIndex.Serialize() - assert.Equal(t, err, nil) - assert.Equal(t, len(blobs), len(copyBlobs)) - // TODO: check key, value and more - - err = index.Delete() - assert.Equal(t, err, nil) - err = copyIndex.Delete() - assert.Equal(t, err, nil) - } -} - -func TestCIndex_Delete(t *testing.T) { - for _, c := range generateTestCases() { - typeParams, indexParams := generateParams(c.indexType, c.metricType) - - index, err := NewCIndex(typeParams, indexParams) - assert.Equal(t, err, nil) - assert.NotEqual(t, index, nil) - - err = index.Delete() - assert.Equal(t, err, nil) - } -} - -func TestCIndex_Error(t *testing.T) { - indexPtr, err := NewCIndex(nil, nil) - assert.Nil(t, err) - - t.Run("Serialize error", func(t *testing.T) { - blobs, err := indexPtr.Serialize() - assert.NotNil(t, err) - assert.Nil(t, blobs) - }) - - t.Run("Load error", func(t *testing.T) { - blobs := []*Blob{{ - Key: "test", - Value: []byte("value"), - }, - } - err = indexPtr.Load(blobs) - assert.NotNil(t, err) - }) - - t.Run("BuildFloatVecIndexWithoutIds error", func(t *testing.T) { - floatVectors := []float32{1.1, 2.2, 3.3} - err = indexPtr.BuildFloatVecIndexWithoutIds(floatVectors) - assert.NotNil(t, err) - }) - - t.Run("BuildBinaryVecIndexWithoutIds error", func(t *testing.T) { - binaryVectors := []byte("binaryVectors") - err = indexPtr.BuildBinaryVecIndexWithoutIds(binaryVectors) - assert.NotNil(t, err) - }) -} diff --git a/internal/indexnode/task.go b/internal/indexnode/task.go index 162216b12f9a8683c513f5132c6b1cf2cd2da93d..52649ad1190d36acedc9a66dda6e0504020b22ce 100644 --- a/internal/indexnode/task.go +++ b/internal/indexnode/task.go @@ -25,6 +25,10 @@ import ( "runtime/debug" "strconv" + "github.com/milvus-io/milvus/internal/proto/schemapb" + + "github.com/milvus-io/milvus/internal/util/indexcgowrapper" + "github.com/milvus-io/milvus/internal/metrics" "go.uber.org/zap" @@ -49,6 +53,8 @@ const ( IndexBuildTaskName = "IndexBuildTask" ) +type Blob = storage.Blob + type task interface { Ctx() context.Context ID() UniqueID // return ReqID @@ -118,8 +124,8 @@ func (bt *BaseTask) Notify(err error) { // IndexBuildTask is used to record the information of the index tasks. type IndexBuildTask struct { BaseTask - index Index cm storage.ChunkManager + index indexcgowrapper.CodecIndex etcdKV *etcdkv.EtcdKV savePaths []string req *indexpb.CreateIndexRequest @@ -326,7 +332,7 @@ func (it *IndexBuildTask) prepareParams(ctx context.Context) error { return nil } -func (it *IndexBuildTask) loadVector(ctx context.Context) (storage.FieldID, storage.FieldData, error) { +func (it *IndexBuildTask) loadFieldData(ctx context.Context) (storage.FieldID, storage.FieldData, error) { getValueByPath := func(path string) ([]byte, error) { data, err := it.cm.Read(path) if err != nil { @@ -370,7 +376,7 @@ func (it *IndexBuildTask) loadVector(ctx context.Context) (storage.FieldID, stor } loadVectorDuration := it.tr.RecordSpan() log.Debug("IndexNode load data success", zap.Int64("buildId", it.req.IndexBuildID)) - it.tr.Record("load vector data done") + it.tr.Record("load field data done") var insertCodec storage.InsertCodec collectionID, partitionID, segmentID, insertData, err2 := insertCodec.DeserializeAll(blobs) @@ -415,33 +421,29 @@ func (it *IndexBuildTask) buildIndex(ctx context.Context) ([]*storage.Blob, erro { var err error var fieldData storage.FieldData - fieldID, fieldData, err = it.loadVector(ctx) + fieldID, fieldData, err = it.loadFieldData(ctx) if err != nil { return nil, err } - floatVectorFieldData, fOk := fieldData.(*storage.FloatVectorFieldData) - if fOk { - err := it.index.BuildFloatVecIndexWithoutIds(floatVectorFieldData.Data) + dataset := indexcgowrapper.GenDataset(fieldData) + dType := dataset.DType + if dType != schemapb.DataType_None { + it.index, err = indexcgowrapper.NewCgoIndex(dType, it.newTypeParams, it.newIndexParams) if err != nil { - log.Error("IndexNode BuildFloatVecIndexWithoutIds failed", zap.Error(err)) + log.Error("failed to create index", zap.Error(err)) return nil, err } - } - binaryVectorFieldData, bOk := fieldData.(*storage.BinaryVectorFieldData) - if bOk { - err := it.index.BuildBinaryVecIndexWithoutIds(binaryVectorFieldData.Data) + + err = it.index.Build(dataset) if err != nil { - log.Error("IndexNode BuildBinaryVecIndexWithoutIds failed", zap.Error(err)) + log.Error("failed to build index", zap.Error(err)) return nil, err } } metrics.IndexNodeKnowhereBuildIndexLatency.WithLabelValues(strconv.FormatInt(Params.IndexNodeCfg.NodeID, 10)).Observe(float64(it.tr.RecordSpan())) - if !fOk && !bOk { - return nil, errors.New("we expect FloatVectorFieldData or BinaryVectorFieldData") - } it.tr.Record("build index done") } @@ -556,25 +558,7 @@ func (it *IndexBuildTask) Execute(ctx context.Context) error { defer it.releaseMemory() var err error - it.index, err = NewCIndex(it.newTypeParams, it.newIndexParams) - if err != nil { - it.SetState(TaskStateFailed) - log.Error("IndexNode IndexBuildTask Execute NewCIndex failed", - zap.Int64("buildId", it.req.IndexBuildID), - zap.Error(err)) - return err - } - - defer func() { - err := it.index.Delete() - if err != nil { - log.Error("IndexNode IndexBuildTask Execute CIndexDelete failed", - zap.Int64("buildId", it.req.IndexBuildID), - zap.Error(err)) - } - }() - it.tr.Record("new CIndex") var blobs []*storage.Blob blobs, err = it.buildIndex(ctx) if err != nil { diff --git a/internal/proto/index_cgo_msg.proto b/internal/proto/index_cgo_msg.proto index ffff8f33d91c8a5fff0a9fcab7eba8f1f26bd8ff..50b1ea5dde5a53c4dcde10970a438318d2e0a224 100644 --- a/internal/proto/index_cgo_msg.proto +++ b/internal/proto/index_cgo_msg.proto @@ -18,6 +18,10 @@ message MapParams { repeated common.KeyValuePair params = 1; } +message MapParamsV2 { + map params = 1; +} + message Binary { string key = 1; bytes value = 2; diff --git a/internal/proto/indexcgopb/index_cgo_msg.pb.go b/internal/proto/indexcgopb/index_cgo_msg.pb.go index 0cb1fb6bdd642a6c05fd080c5c2c578efd3bb2ac..217d9cd1b019cb414a9e0805daf9b58067cae6f7 100644 --- a/internal/proto/indexcgopb/index_cgo_msg.pb.go +++ b/internal/proto/indexcgopb/index_cgo_msg.pb.go @@ -139,6 +139,45 @@ func (m *MapParams) GetParams() []*commonpb.KeyValuePair { return nil } +type MapParamsV2 struct { + Params map[string]string `protobuf:"bytes,1,rep,name=params,proto3" json:"params,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *MapParamsV2) Reset() { *m = MapParamsV2{} } +func (m *MapParamsV2) String() string { return proto.CompactTextString(m) } +func (*MapParamsV2) ProtoMessage() {} +func (*MapParamsV2) Descriptor() ([]byte, []int) { + return fileDescriptor_c009bd9544a7343c, []int{3} +} + +func (m *MapParamsV2) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_MapParamsV2.Unmarshal(m, b) +} +func (m *MapParamsV2) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_MapParamsV2.Marshal(b, m, deterministic) +} +func (m *MapParamsV2) XXX_Merge(src proto.Message) { + xxx_messageInfo_MapParamsV2.Merge(m, src) +} +func (m *MapParamsV2) XXX_Size() int { + return xxx_messageInfo_MapParamsV2.Size(m) +} +func (m *MapParamsV2) XXX_DiscardUnknown() { + xxx_messageInfo_MapParamsV2.DiscardUnknown(m) +} + +var xxx_messageInfo_MapParamsV2 proto.InternalMessageInfo + +func (m *MapParamsV2) GetParams() map[string]string { + if m != nil { + return m.Params + } + return nil +} + type Binary struct { Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` @@ -151,7 +190,7 @@ func (m *Binary) Reset() { *m = Binary{} } func (m *Binary) String() string { return proto.CompactTextString(m) } func (*Binary) ProtoMessage() {} func (*Binary) Descriptor() ([]byte, []int) { - return fileDescriptor_c009bd9544a7343c, []int{3} + return fileDescriptor_c009bd9544a7343c, []int{4} } func (m *Binary) XXX_Unmarshal(b []byte) error { @@ -197,7 +236,7 @@ func (m *BinarySet) Reset() { *m = BinarySet{} } func (m *BinarySet) String() string { return proto.CompactTextString(m) } func (*BinarySet) ProtoMessage() {} func (*BinarySet) Descriptor() ([]byte, []int) { - return fileDescriptor_c009bd9544a7343c, []int{4} + return fileDescriptor_c009bd9544a7343c, []int{5} } func (m *BinarySet) XXX_Unmarshal(b []byte) error { @@ -229,6 +268,8 @@ func init() { proto.RegisterType((*TypeParams)(nil), "milvus.proto.indexcgo.TypeParams") proto.RegisterType((*IndexParams)(nil), "milvus.proto.indexcgo.IndexParams") proto.RegisterType((*MapParams)(nil), "milvus.proto.indexcgo.MapParams") + proto.RegisterType((*MapParamsV2)(nil), "milvus.proto.indexcgo.MapParamsV2") + proto.RegisterMapType((map[string]string)(nil), "milvus.proto.indexcgo.MapParamsV2.ParamsEntry") proto.RegisterType((*Binary)(nil), "milvus.proto.indexcgo.Binary") proto.RegisterType((*BinarySet)(nil), "milvus.proto.indexcgo.BinarySet") } @@ -236,21 +277,24 @@ func init() { func init() { proto.RegisterFile("index_cgo_msg.proto", fileDescriptor_c009bd9544a7343c) } var fileDescriptor_c009bd9544a7343c = []byte{ - // 250 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x12, 0xce, 0xcc, 0x4b, 0x49, - 0xad, 0x88, 0x4f, 0x4e, 0xcf, 0x8f, 0xcf, 0x2d, 0x4e, 0xd7, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, - 0x12, 0xcd, 0xcd, 0xcc, 0x29, 0x2b, 0x2d, 0x86, 0xf0, 0xf4, 0xc0, 0x2a, 0x92, 0xd3, 0xf3, 0xa5, - 0x78, 0x92, 0xf3, 0x73, 0x73, 0xf3, 0xf3, 0x20, 0xc2, 0x4a, 0xee, 0x5c, 0x5c, 0x21, 0x95, 0x05, - 0xa9, 0x01, 0x89, 0x45, 0x89, 0xb9, 0xc5, 0x42, 0x96, 0x5c, 0x6c, 0x05, 0x60, 0x96, 0x04, 0xa3, - 0x02, 0xb3, 0x06, 0xb7, 0x91, 0xa2, 0x1e, 0x8a, 0x19, 0x50, 0x9d, 0xde, 0xa9, 0x95, 0x61, 0x89, - 0x39, 0xa5, 0xa9, 0x01, 0x89, 0x99, 0x45, 0x41, 0x50, 0x0d, 0x4a, 0x1e, 0x5c, 0xdc, 0x9e, 0x20, - 0x2b, 0x28, 0x37, 0xc9, 0x8d, 0x8b, 0xd3, 0x37, 0xb1, 0x80, 0x72, 0x73, 0x0c, 0xb8, 0xd8, 0x9c, - 0x32, 0xf3, 0x12, 0x8b, 0x2a, 0x85, 0x04, 0xb8, 0x98, 0xb3, 0x53, 0x2b, 0x25, 0x18, 0x15, 0x18, - 0x35, 0x38, 0x83, 0x40, 0x4c, 0x21, 0x11, 0x2e, 0xd6, 0x32, 0x90, 0x06, 0x09, 0x26, 0x05, 0x46, - 0x0d, 0x9e, 0x20, 0x08, 0x47, 0xc9, 0x81, 0x8b, 0x13, 0xa2, 0x23, 0x38, 0xb5, 0x44, 0xc8, 0x98, - 0x8b, 0x35, 0x25, 0xb1, 0x24, 0x11, 0x66, 0xb1, 0xac, 0x1e, 0xd6, 0xe0, 0xd4, 0x83, 0x68, 0x08, - 0x82, 0xa8, 0x75, 0x32, 0x8f, 0x32, 0x4d, 0xcf, 0x2c, 0xc9, 0x28, 0x4d, 0x02, 0xb9, 0x4c, 0x1f, - 0xa2, 0x43, 0x37, 0x33, 0x1f, 0xca, 0xd2, 0xcf, 0xcc, 0x2b, 0x49, 0x2d, 0xca, 0x4b, 0xcc, 0xd1, - 0x07, 0x1b, 0xa2, 0x0f, 0x33, 0xa4, 0x20, 0x29, 0x89, 0x0d, 0x2c, 0x62, 0x0c, 0x08, 0x00, 0x00, - 0xff, 0xff, 0xaf, 0x1f, 0x55, 0x04, 0xca, 0x01, 0x00, 0x00, + // 289 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x90, 0x41, 0x4b, 0xc3, 0x30, + 0x14, 0xc7, 0xe9, 0xc6, 0x0a, 0x7d, 0xdd, 0x41, 0xa2, 0x42, 0x19, 0x08, 0xb3, 0xa7, 0x5d, 0x4c, + 0x65, 0x43, 0x74, 0x9e, 0x64, 0xe0, 0x54, 0x44, 0x18, 0x55, 0x76, 0xf0, 0x32, 0xd2, 0x2e, 0xd4, + 0x60, 0x9b, 0x94, 0x34, 0x1d, 0xf6, 0x5b, 0xf8, 0x91, 0xa5, 0x49, 0x2b, 0x53, 0x94, 0x1d, 0x76, + 0xfb, 0xe7, 0xcf, 0xfb, 0xfd, 0xda, 0xf7, 0xe0, 0x90, 0xf1, 0x35, 0xfd, 0x58, 0xc5, 0x89, 0x58, + 0x65, 0x45, 0x82, 0x73, 0x29, 0x94, 0x40, 0xc7, 0x19, 0x4b, 0x37, 0x65, 0x61, 0x5e, 0x58, 0x4f, + 0xc4, 0x89, 0x18, 0xf4, 0x63, 0x91, 0x65, 0x82, 0x9b, 0xda, 0xbf, 0x03, 0x78, 0xa9, 0x72, 0xba, + 0x20, 0x92, 0x64, 0x05, 0x9a, 0x82, 0x9d, 0xeb, 0xe4, 0x59, 0xc3, 0xee, 0xc8, 0x1d, 0x9f, 0xe2, + 0x1f, 0x8e, 0x86, 0x7c, 0xa4, 0xd5, 0x92, 0xa4, 0x25, 0x5d, 0x10, 0x26, 0xc3, 0x06, 0xf0, 0xef, + 0xc1, 0x7d, 0xa8, 0x3f, 0xb1, 0xbf, 0x69, 0x0e, 0xce, 0x13, 0xc9, 0xf7, 0xf7, 0x7c, 0x5a, 0xe0, + 0x7e, 0x8b, 0x96, 0x63, 0x34, 0xff, 0xa5, 0xc2, 0xf8, 0xcf, 0x03, 0xe1, 0x2d, 0x06, 0x9b, 0x70, + 0xcb, 0x95, 0xac, 0x5a, 0xef, 0x60, 0x0a, 0xee, 0x56, 0x8d, 0x0e, 0xa0, 0xfb, 0x4e, 0x2b, 0xcf, + 0x1a, 0x5a, 0x23, 0x27, 0xac, 0x23, 0x3a, 0x82, 0xde, 0xa6, 0xfe, 0x1b, 0xaf, 0xa3, 0x3b, 0xf3, + 0xb8, 0xee, 0x5c, 0x59, 0xfe, 0x39, 0xd8, 0x33, 0xc6, 0xc9, 0x6e, 0xaa, 0xdf, 0x50, 0xfe, 0x0d, + 0x38, 0x86, 0x78, 0xa6, 0x0a, 0x4d, 0xa0, 0xb7, 0x26, 0x8a, 0xb4, 0x0b, 0x9c, 0xfc, 0xb3, 0x80, + 0x01, 0x42, 0x33, 0x3b, 0xbb, 0x7c, 0xbd, 0x48, 0x98, 0x7a, 0x2b, 0xa3, 0xfa, 0x58, 0x81, 0x21, + 0xce, 0x98, 0x68, 0x52, 0xc0, 0xb8, 0xa2, 0x92, 0x93, 0x34, 0xd0, 0x92, 0xa0, 0x95, 0xe4, 0x51, + 0x64, 0xeb, 0x66, 0xf2, 0x15, 0x00, 0x00, 0xff, 0xff, 0x83, 0x12, 0x13, 0xfb, 0x5d, 0x02, 0x00, + 0x00, } diff --git a/internal/proto/milvus.proto b/internal/proto/milvus.proto index b9969438acda37efe4b17cfc16e8c53a87f30703..ba1f8a1674e448687eecb130ed2eb01d1c95c989 100644 --- a/internal/proto/milvus.proto +++ b/internal/proto/milvus.proto @@ -440,7 +440,9 @@ message CreateIndexRequest { // The vector field name in this particular collection string field_name = 4; // Support keys: index_type,metric_type, params. Different index_type may has different params. - repeated common.KeyValuePair extra_params = 5; + repeated common.KeyValuePair extra_params = 5; + // Version before 2.0.2 doesn't contain index_name, we use default index name. + string index_name = 6; } /* diff --git a/internal/proto/milvuspb/milvus.pb.go b/internal/proto/milvuspb/milvus.pb.go index 11751f2d40233d6323c127de4d71376274f66b65..0aa2768d8c3deb0eece36a4ddb1d82cc5d3d4038 100644 --- a/internal/proto/milvuspb/milvus.pb.go +++ b/internal/proto/milvuspb/milvus.pb.go @@ -2068,10 +2068,12 @@ type CreateIndexRequest struct { // The vector field name in this particular collection FieldName string `protobuf:"bytes,4,opt,name=field_name,json=fieldName,proto3" json:"field_name,omitempty"` // Support keys: index_type,metric_type, params. Different index_type may has different params. - ExtraParams []*commonpb.KeyValuePair `protobuf:"bytes,5,rep,name=extra_params,json=extraParams,proto3" json:"extra_params,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + ExtraParams []*commonpb.KeyValuePair `protobuf:"bytes,5,rep,name=extra_params,json=extraParams,proto3" json:"extra_params,omitempty"` + // Version before 2.0.2 doesn't contain index_name, we use default index name. + IndexName string `protobuf:"bytes,6,opt,name=index_name,json=indexName,proto3" json:"index_name,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *CreateIndexRequest) Reset() { *m = CreateIndexRequest{} } @@ -2134,6 +2136,13 @@ func (m *CreateIndexRequest) GetExtraParams() []*commonpb.KeyValuePair { return nil } +func (m *CreateIndexRequest) GetIndexName() string { + if m != nil { + return m.IndexName + } + return "" +} + // // Get created index information. // Current release of Milvus only supports showing latest built index. @@ -5262,245 +5271,245 @@ func init() { func init() { proto.RegisterFile("milvus.proto", fileDescriptor_02345ba45cc0e303) } var fileDescriptor_02345ba45cc0e303 = []byte{ - // 3797 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x3b, 0x4b, 0x6f, 0x1c, 0x47, - 0x73, 0x9c, 0x5d, 0xee, 0xab, 0x76, 0x97, 0x5c, 0x36, 0x29, 0x6a, 0xb5, 0x7a, 0x51, 0xf3, 0x59, - 0x9f, 0x28, 0xe9, 0x93, 0xf4, 0x89, 0xf2, 0xe3, 0x8b, 0x9c, 0xc4, 0x96, 0xc8, 0x58, 0x22, 0x2c, - 0x29, 0xf4, 0xd0, 0x76, 0xe0, 0x18, 0xc2, 0x60, 0xb8, 0xd3, 0x5c, 0x0e, 0x38, 0x3b, 0xb3, 0x9e, - 0xee, 0x15, 0x45, 0x9f, 0x0c, 0x38, 0x48, 0x10, 0xd8, 0xb1, 0x11, 0x24, 0xc8, 0xe3, 0x90, 0x1c, - 0xf2, 0x38, 0xe4, 0x90, 0x20, 0x8e, 0x83, 0x24, 0xc8, 0x25, 0x39, 0x04, 0x48, 0x0e, 0x01, 0xf2, - 0xb8, 0xe4, 0x90, 0x4b, 0xfe, 0x80, 0x6f, 0x39, 0xe6, 0x10, 0xf4, 0x63, 0x66, 0x67, 0x66, 0x7b, - 0x96, 0x4b, 0xad, 0xf5, 0x91, 0xbc, 0xcd, 0x54, 0x57, 0x75, 0x57, 0x57, 0xd7, 0xa3, 0xbb, 0xba, - 0x1a, 0x6a, 0x5d, 0xc7, 0x7d, 0xd6, 0x27, 0x37, 0x7b, 0x81, 0x4f, 0x7d, 0x34, 0x1f, 0xff, 0xbb, - 0x29, 0x7e, 0x5a, 0xb5, 0xb6, 0xdf, 0xed, 0xfa, 0x9e, 0x00, 0xb6, 0x6a, 0xa4, 0xbd, 0x83, 0xbb, - 0x96, 0xf8, 0xd3, 0xff, 0x50, 0x03, 0xb4, 0x1a, 0x60, 0x8b, 0xe2, 0x7b, 0xae, 0x63, 0x11, 0x03, - 0x7f, 0xd2, 0xc7, 0x84, 0xa2, 0x1f, 0xc3, 0xf4, 0x96, 0x45, 0x70, 0x53, 0x5b, 0xd2, 0x96, 0xab, - 0x2b, 0xe7, 0x6e, 0x26, 0xba, 0x95, 0xdd, 0x3d, 0x26, 0x9d, 0xfb, 0x16, 0xc1, 0x06, 0xc7, 0x44, - 0xa7, 0xa1, 0x64, 0x6f, 0x99, 0x9e, 0xd5, 0xc5, 0xcd, 0xdc, 0x92, 0xb6, 0x5c, 0x31, 0x8a, 0xf6, - 0xd6, 0x13, 0xab, 0x8b, 0xd1, 0x15, 0x98, 0x6d, 0xfb, 0xae, 0x8b, 0xdb, 0xd4, 0xf1, 0x3d, 0x81, - 0x90, 0xe7, 0x08, 0x33, 0x03, 0x30, 0x47, 0x5c, 0x80, 0x82, 0xc5, 0x78, 0x68, 0x4e, 0xf3, 0x66, - 0xf1, 0xa3, 0x13, 0x68, 0xac, 0x05, 0x7e, 0xef, 0x65, 0x71, 0x17, 0x0d, 0x9a, 0x8f, 0x0f, 0xfa, - 0x07, 0x1a, 0xcc, 0xdd, 0x73, 0x29, 0x0e, 0x8e, 0xa9, 0x50, 0x7e, 0x2f, 0x07, 0xa7, 0xc5, 0xaa, - 0xad, 0x46, 0xe8, 0x47, 0xc9, 0xe5, 0x22, 0x14, 0x85, 0x56, 0x71, 0x36, 0x6b, 0x86, 0xfc, 0x43, - 0xe7, 0x01, 0xc8, 0x8e, 0x15, 0xd8, 0xc4, 0xf4, 0xfa, 0xdd, 0x66, 0x61, 0x49, 0x5b, 0x2e, 0x18, - 0x15, 0x01, 0x79, 0xd2, 0xef, 0x22, 0x03, 0xe6, 0xda, 0xbe, 0x47, 0x1c, 0x42, 0xb1, 0xd7, 0xde, - 0x37, 0x5d, 0xfc, 0x0c, 0xbb, 0xcd, 0xe2, 0x92, 0xb6, 0x3c, 0xb3, 0x72, 0x59, 0xc9, 0xf7, 0xea, - 0x00, 0xfb, 0x11, 0x43, 0x36, 0x1a, 0xed, 0x14, 0x44, 0xff, 0x42, 0x83, 0x53, 0x4c, 0x61, 0x8e, - 0x85, 0x60, 0xf4, 0x3f, 0xd3, 0x60, 0xe1, 0xa1, 0x45, 0x8e, 0xc7, 0x2a, 0x9d, 0x07, 0xa0, 0x4e, - 0x17, 0x9b, 0x84, 0x5a, 0xdd, 0x1e, 0x5f, 0xa9, 0x69, 0xa3, 0xc2, 0x20, 0x9b, 0x0c, 0xa0, 0x7f, - 0x04, 0xb5, 0xfb, 0xbe, 0xef, 0x1a, 0x98, 0xf4, 0x7c, 0x8f, 0x60, 0x74, 0x07, 0x8a, 0x84, 0x5a, - 0xb4, 0x4f, 0x24, 0x93, 0x67, 0x95, 0x4c, 0x6e, 0x72, 0x14, 0x43, 0xa2, 0x32, 0x7d, 0x7d, 0x66, - 0xb9, 0x7d, 0xc1, 0x63, 0xd9, 0x10, 0x3f, 0xfa, 0xc7, 0x30, 0xb3, 0x49, 0x03, 0xc7, 0xeb, 0x7c, - 0x8f, 0x9d, 0x57, 0xc2, 0xce, 0xff, 0x53, 0x83, 0x33, 0x6b, 0x98, 0xb4, 0x03, 0x67, 0xeb, 0x98, - 0x98, 0x83, 0x0e, 0xb5, 0x01, 0x64, 0x7d, 0x8d, 0x8b, 0x3a, 0x6f, 0x24, 0x60, 0xa9, 0xc5, 0x28, - 0xa4, 0x17, 0xe3, 0xb3, 0x02, 0xb4, 0x54, 0x93, 0x9a, 0x44, 0x7c, 0x3f, 0x17, 0x59, 0x69, 0x8e, - 0x13, 0xa5, 0x6c, 0x4c, 0xc6, 0x85, 0xc1, 0x68, 0x9b, 0x1c, 0x10, 0x19, 0x73, 0x7a, 0x56, 0x79, - 0xc5, 0xac, 0x56, 0xe0, 0xd4, 0x33, 0x27, 0xa0, 0x7d, 0xcb, 0x35, 0xdb, 0x3b, 0x96, 0xe7, 0x61, - 0x97, 0xcb, 0x89, 0xb9, 0xaf, 0xfc, 0x72, 0xc5, 0x98, 0x97, 0x8d, 0xab, 0xa2, 0x8d, 0x09, 0x8b, - 0xa0, 0x57, 0x61, 0xb1, 0xb7, 0xb3, 0x4f, 0x9c, 0xf6, 0x10, 0x51, 0x81, 0x13, 0x2d, 0x84, 0xad, - 0x09, 0xaa, 0xeb, 0x30, 0xd7, 0xe6, 0x1e, 0xd0, 0x36, 0x99, 0xd4, 0x84, 0x18, 0x8b, 0x5c, 0x8c, - 0x0d, 0xd9, 0xf0, 0x7e, 0x08, 0x67, 0x6c, 0x85, 0xc8, 0x7d, 0xda, 0x8e, 0x11, 0x94, 0x38, 0xc1, - 0xbc, 0x6c, 0xfc, 0x80, 0xb6, 0x07, 0x34, 0x49, 0xdf, 0x55, 0x4e, 0xfb, 0xae, 0x26, 0x94, 0xb8, - 0x2f, 0xc6, 0xa4, 0x59, 0xe1, 0x6c, 0x86, 0xbf, 0x68, 0x1d, 0x66, 0x09, 0xb5, 0x02, 0x6a, 0xf6, - 0x7c, 0xe2, 0x30, 0xb9, 0x90, 0x26, 0x2c, 0xe5, 0x97, 0xab, 0x2b, 0x4b, 0xca, 0x45, 0x7a, 0x17, - 0xef, 0xaf, 0x59, 0xd4, 0xda, 0xb0, 0x9c, 0xc0, 0x98, 0xe1, 0x84, 0x1b, 0x21, 0x9d, 0xda, 0x41, - 0x56, 0x27, 0x72, 0x90, 0x2a, 0x2d, 0xae, 0x29, 0x7d, 0x17, 0xf3, 0xa4, 0x8f, 0x7c, 0xcb, 0x3e, - 0x1e, 0x9e, 0xf4, 0x2b, 0x0d, 0x9a, 0x06, 0x76, 0xb1, 0x45, 0x8e, 0x87, 0x91, 0xeb, 0xbf, 0xad, - 0xc1, 0x85, 0x07, 0x98, 0xc6, 0xcc, 0x85, 0x5a, 0xd4, 0x21, 0xd4, 0x69, 0x1f, 0xe5, 0x86, 0x41, - 0xff, 0x5a, 0x83, 0x8b, 0x99, 0x6c, 0x4d, 0xe2, 0x3d, 0xde, 0x80, 0x02, 0xfb, 0x22, 0xcd, 0x1c, - 0x57, 0xe6, 0x4b, 0x59, 0xca, 0xfc, 0x21, 0x73, 0xca, 0x5c, 0x9b, 0x05, 0xbe, 0xfe, 0x3f, 0x1a, - 0x2c, 0x6e, 0xee, 0xf8, 0x7b, 0x03, 0x96, 0x5e, 0x86, 0x80, 0x92, 0xfe, 0x34, 0x9f, 0xf2, 0xa7, - 0xe8, 0x36, 0x4c, 0xd3, 0xfd, 0x1e, 0xe6, 0xae, 0x78, 0x66, 0xe5, 0xfc, 0x4d, 0xc5, 0x3e, 0xf9, - 0x26, 0x63, 0xf2, 0xfd, 0xfd, 0x1e, 0x36, 0x38, 0x2a, 0xba, 0x0a, 0x8d, 0x94, 0xc8, 0x43, 0x8f, - 0x34, 0x9b, 0x94, 0x39, 0xd1, 0xff, 0x2e, 0x07, 0xa7, 0x87, 0xa6, 0x38, 0x89, 0xb0, 0x55, 0x63, - 0xe7, 0x94, 0x63, 0xa3, 0xcb, 0x10, 0x53, 0x01, 0xd3, 0xb1, 0xd9, 0x56, 0x36, 0xbf, 0x9c, 0x37, - 0xea, 0x31, 0xc7, 0x6c, 0x13, 0x74, 0x03, 0xd0, 0x90, 0xbf, 0x14, 0x6e, 0x79, 0xda, 0x98, 0x4b, - 0x3b, 0x4c, 0xee, 0x94, 0x95, 0x1e, 0x53, 0x88, 0x60, 0xda, 0x58, 0x50, 0xb8, 0x4c, 0x82, 0x6e, - 0xc3, 0x82, 0xe3, 0x3d, 0xc6, 0x5d, 0x3f, 0xd8, 0x37, 0x7b, 0x38, 0x68, 0x63, 0x8f, 0x5a, 0x1d, - 0x4c, 0x9a, 0x45, 0xce, 0xd1, 0x7c, 0xd8, 0xb6, 0x31, 0x68, 0xd2, 0xbf, 0xd5, 0x60, 0x51, 0x6c, - 0x65, 0x37, 0xac, 0x80, 0x3a, 0x47, 0x1d, 0xba, 0x2f, 0xc3, 0x4c, 0x2f, 0xe4, 0x43, 0xe0, 0x89, - 0x8d, 0x77, 0x3d, 0x82, 0x72, 0x2b, 0xfb, 0x46, 0x83, 0x05, 0xb6, 0xcb, 0x3c, 0x49, 0x3c, 0xff, - 0xa5, 0x06, 0xf3, 0x0f, 0x2d, 0x72, 0x92, 0x58, 0xfe, 0x6b, 0x19, 0x82, 0x22, 0x9e, 0x8f, 0xf4, - 0x2c, 0x76, 0x05, 0x66, 0x93, 0x4c, 0x87, 0xdb, 0x9a, 0x99, 0x04, 0xd7, 0x44, 0xff, 0xdb, 0x41, - 0xac, 0x3a, 0x61, 0x9c, 0xff, 0xbd, 0x06, 0xe7, 0x1f, 0x60, 0x1a, 0x71, 0x7d, 0x2c, 0x62, 0xda, - 0xb8, 0xda, 0xf2, 0x95, 0x88, 0xc8, 0x4a, 0xe6, 0x8f, 0x24, 0xf2, 0x7d, 0x91, 0x83, 0x53, 0x2c, - 0x2c, 0x1c, 0x0f, 0x25, 0x18, 0xe7, 0x54, 0xa2, 0x50, 0x94, 0x82, 0x4a, 0x51, 0xa2, 0x78, 0x5a, - 0x1c, 0x3b, 0x9e, 0xea, 0x7f, 0x95, 0x13, 0xfb, 0x80, 0xb8, 0x34, 0x26, 0x59, 0x16, 0x05, 0xaf, - 0x39, 0x25, 0xaf, 0x3a, 0xd4, 0x22, 0xc8, 0xfa, 0x5a, 0x18, 0x1f, 0x13, 0xb0, 0x63, 0x1b, 0x1e, - 0xbf, 0xd4, 0x60, 0x31, 0x3c, 0x07, 0x6e, 0xe2, 0x4e, 0x17, 0x7b, 0xf4, 0xc5, 0x75, 0x28, 0xad, - 0x01, 0x39, 0x85, 0x06, 0x9c, 0x83, 0x0a, 0x11, 0xe3, 0x44, 0x47, 0xbc, 0x01, 0x40, 0xff, 0x07, - 0x0d, 0x4e, 0x0f, 0xb1, 0x33, 0xc9, 0x22, 0x36, 0xa1, 0xe4, 0x78, 0x36, 0x7e, 0x1e, 0x71, 0x13, - 0xfe, 0xb2, 0x96, 0xad, 0xbe, 0xe3, 0xda, 0x11, 0x1b, 0xe1, 0x2f, 0xba, 0x04, 0x35, 0xec, 0x59, - 0x5b, 0x2e, 0x36, 0x39, 0x2e, 0x57, 0xe4, 0xb2, 0x51, 0x15, 0xb0, 0x75, 0x06, 0x62, 0xc4, 0xdb, - 0x0e, 0xe6, 0xc4, 0x05, 0x41, 0x2c, 0x7f, 0xf5, 0xdf, 0xd0, 0x60, 0x9e, 0x69, 0xa1, 0xe4, 0x9e, - 0xbc, 0x5c, 0x69, 0x2e, 0x41, 0x35, 0xa6, 0x66, 0x72, 0x22, 0x71, 0x90, 0xbe, 0x0b, 0x0b, 0x49, - 0x76, 0x26, 0x91, 0xe6, 0x05, 0x80, 0x68, 0xad, 0x84, 0x35, 0xe4, 0x8d, 0x18, 0x44, 0xff, 0x2e, - 0xca, 0xf6, 0x72, 0x31, 0x1d, 0x71, 0x32, 0x8a, 0x2f, 0x49, 0xdc, 0x9f, 0x57, 0x38, 0x84, 0x37, - 0xaf, 0x41, 0x0d, 0x3f, 0xa7, 0x81, 0x65, 0xf6, 0xac, 0xc0, 0xea, 0x0a, 0xb3, 0x1a, 0xcb, 0xf5, - 0x56, 0x39, 0xd9, 0x06, 0xa7, 0xd2, 0xff, 0x85, 0x6d, 0xd3, 0xa4, 0xba, 0x1e, 0xf7, 0x19, 0x9f, - 0x07, 0xe0, 0xea, 0x2c, 0x9a, 0x0b, 0xa2, 0x99, 0x43, 0x78, 0x70, 0xfb, 0x53, 0x0d, 0x1a, 0x7c, - 0x0a, 0x62, 0x3e, 0x3d, 0xd6, 0x6d, 0x8a, 0x46, 0x4b, 0xd1, 0x8c, 0x30, 0xae, 0x9f, 0x81, 0xa2, - 0x14, 0x6c, 0x7e, 0x5c, 0xc1, 0x4a, 0x82, 0x03, 0xa6, 0xa1, 0xff, 0x91, 0x06, 0xa7, 0x52, 0x22, - 0x9f, 0x44, 0xa3, 0xdf, 0x07, 0x24, 0x66, 0x68, 0x0f, 0xa6, 0x1d, 0x06, 0xe2, 0xcb, 0xca, 0xa8, - 0x93, 0x16, 0x92, 0x31, 0xe7, 0xa4, 0x20, 0x44, 0xff, 0x77, 0x0d, 0xce, 0x3d, 0xc0, 0x94, 0xa3, - 0xde, 0x67, 0x5e, 0x65, 0x23, 0xf0, 0x3b, 0x01, 0x26, 0xe4, 0xe4, 0xea, 0xc7, 0xef, 0x88, 0x9d, - 0x9b, 0x6a, 0x4a, 0x93, 0xc8, 0xff, 0x12, 0xd4, 0xf8, 0x18, 0xd8, 0x36, 0x03, 0x7f, 0x8f, 0x48, - 0x3d, 0xaa, 0x4a, 0x98, 0xe1, 0xef, 0x71, 0x85, 0xa0, 0x3e, 0xb5, 0x5c, 0x81, 0x20, 0x43, 0x06, - 0x87, 0xb0, 0x66, 0x6e, 0x83, 0x21, 0x63, 0xac, 0x73, 0x7c, 0x72, 0x65, 0xfc, 0x27, 0x1a, 0x9c, - 0x4a, 0x4d, 0x65, 0x12, 0xd9, 0xbe, 0x26, 0xf6, 0x95, 0x62, 0x32, 0x33, 0x2b, 0x17, 0x95, 0x34, - 0xb1, 0xc1, 0x04, 0x36, 0xba, 0x08, 0xd5, 0x6d, 0xcb, 0x71, 0xcd, 0x00, 0x5b, 0xc4, 0xf7, 0xe4, - 0x44, 0x81, 0x81, 0x0c, 0x0e, 0xd1, 0xff, 0x49, 0x13, 0x77, 0x66, 0x27, 0xdc, 0xe3, 0xfd, 0x71, - 0x0e, 0xea, 0xeb, 0x1e, 0xc1, 0x01, 0x3d, 0xfe, 0x67, 0x0f, 0xf4, 0x16, 0x54, 0xf9, 0xc4, 0x88, - 0x69, 0x5b, 0xd4, 0x92, 0xe1, 0xea, 0x82, 0x32, 0xc1, 0xfe, 0x0e, 0xc3, 0x5b, 0xb3, 0xa8, 0x65, - 0x08, 0xe9, 0x10, 0xf6, 0x8d, 0xce, 0x42, 0x65, 0xc7, 0x22, 0x3b, 0xe6, 0x2e, 0xde, 0x17, 0x1b, - 0xc2, 0xba, 0x51, 0x66, 0x80, 0x77, 0xf1, 0x3e, 0x41, 0x67, 0xa0, 0xec, 0xf5, 0xbb, 0xc2, 0xc0, - 0x4a, 0x4b, 0xda, 0x72, 0xdd, 0x28, 0x79, 0xfd, 0x2e, 0x37, 0xaf, 0x7f, 0xcd, 0xc1, 0xcc, 0xe3, - 0x3e, 0x3b, 0xe9, 0xf0, 0xeb, 0x81, 0xbe, 0x4b, 0x5f, 0x4c, 0x19, 0xaf, 0x41, 0x5e, 0xec, 0x19, - 0x18, 0x45, 0x53, 0xc9, 0xf8, 0xfa, 0x1a, 0x31, 0x18, 0x12, 0x4f, 0x8d, 0xf7, 0xdb, 0x6d, 0xb9, - 0xfd, 0xca, 0x73, 0x66, 0x2b, 0x0c, 0x22, 0x36, 0x5f, 0x67, 0xa1, 0x82, 0x83, 0x20, 0xda, 0x9c, - 0xf1, 0xa9, 0xe0, 0x20, 0x10, 0x8d, 0x3a, 0xd4, 0xac, 0xf6, 0xae, 0xe7, 0xef, 0xb9, 0xd8, 0xee, - 0x60, 0x9b, 0x2f, 0x7b, 0xd9, 0x48, 0xc0, 0x84, 0x62, 0xb0, 0x85, 0x37, 0xdb, 0x1e, 0xe5, 0x47, - 0x8c, 0x3c, 0x53, 0x0c, 0x06, 0x59, 0xf5, 0x28, 0x6b, 0xb6, 0xb1, 0x8b, 0x29, 0xe6, 0xcd, 0x25, - 0xd1, 0x2c, 0x20, 0xb2, 0xb9, 0xdf, 0x8b, 0xa8, 0xcb, 0xa2, 0x59, 0x40, 0x58, 0xf3, 0x39, 0xa8, - 0x0c, 0xf2, 0xff, 0x95, 0x41, 0x9e, 0x90, 0x03, 0xf4, 0xff, 0xd6, 0xa0, 0xbe, 0xc6, 0xbb, 0x3a, - 0x01, 0x4a, 0x87, 0x60, 0x1a, 0x3f, 0xef, 0x05, 0xd2, 0x74, 0xf8, 0xf7, 0x48, 0x3d, 0xd2, 0x9f, - 0x41, 0x63, 0xc3, 0xb5, 0xda, 0x78, 0xc7, 0x77, 0x6d, 0x1c, 0xf0, 0xd8, 0x8e, 0x1a, 0x90, 0xa7, - 0x56, 0x47, 0x6e, 0x1e, 0xd8, 0x27, 0xfa, 0x89, 0x3c, 0xdb, 0x09, 0xb7, 0xf4, 0x8a, 0x32, 0xca, - 0xc6, 0xba, 0x89, 0xa5, 0x4c, 0x17, 0xa1, 0xc8, 0xef, 0xe4, 0xc4, 0xb6, 0xa2, 0x66, 0xc8, 0x3f, - 0xfd, 0x69, 0x62, 0xdc, 0x07, 0x81, 0xdf, 0xef, 0xa1, 0x75, 0xa8, 0xf5, 0x06, 0x30, 0xa6, 0xab, - 0xd9, 0x31, 0x3d, 0xcd, 0xb4, 0x91, 0x20, 0xd5, 0xbf, 0xcb, 0x43, 0x7d, 0x13, 0x5b, 0x41, 0x7b, - 0xe7, 0x24, 0x24, 0x59, 0x98, 0xc4, 0x6d, 0xe2, 0xca, 0x55, 0x63, 0x9f, 0xe8, 0x3a, 0xcc, 0xc5, - 0x26, 0x64, 0x76, 0x98, 0x80, 0xb8, 0xde, 0xd7, 0x8c, 0x46, 0x2f, 0x2d, 0xb8, 0x37, 0xa0, 0x6c, - 0x13, 0xd7, 0xe4, 0x4b, 0x54, 0xe2, 0x4b, 0xa4, 0x9e, 0xdf, 0x1a, 0x71, 0xf9, 0xd2, 0x94, 0x6c, - 0xf1, 0x81, 0x7e, 0x00, 0x75, 0xbf, 0x4f, 0x7b, 0x7d, 0x6a, 0x0a, 0xbf, 0xd3, 0x2c, 0x73, 0xf6, - 0x6a, 0x02, 0xc8, 0xdd, 0x12, 0x41, 0xef, 0x40, 0x9d, 0x70, 0x51, 0x86, 0x3b, 0xef, 0xca, 0xb8, - 0x1b, 0xc4, 0x9a, 0xa0, 0x13, 0x5b, 0x6f, 0x74, 0x15, 0x1a, 0x34, 0xb0, 0x9e, 0x61, 0x37, 0x76, - 0xdb, 0x06, 0xdc, 0xda, 0x66, 0x05, 0x7c, 0x70, 0xd3, 0x76, 0x0b, 0xe6, 0x3b, 0x7d, 0x2b, 0xb0, - 0x3c, 0x8a, 0x71, 0x0c, 0xbb, 0xca, 0xb1, 0x51, 0xd4, 0x14, 0x11, 0xe8, 0xef, 0xc2, 0xf4, 0x43, - 0x87, 0x72, 0x41, 0x32, 0x9f, 0xa5, 0xf1, 0x73, 0x0e, 0xf7, 0x4c, 0x67, 0xa0, 0x1c, 0xf8, 0x7b, - 0xc2, 0x07, 0xe7, 0xb8, 0x0a, 0x96, 0x02, 0x7f, 0x8f, 0x3b, 0x58, 0x5e, 0xa3, 0xe0, 0x07, 0x52, - 0x37, 0x73, 0x86, 0xfc, 0xd3, 0xff, 0x42, 0x1b, 0x28, 0x0f, 0x73, 0x9f, 0xe4, 0xc5, 0xfc, 0xe7, - 0x5b, 0x50, 0x0a, 0x04, 0xfd, 0xc8, 0xdb, 0xd5, 0xf8, 0x48, 0x3c, 0x06, 0x84, 0x54, 0xe3, 0xdf, - 0xf0, 0xfc, 0x8a, 0x06, 0xb5, 0x77, 0xdc, 0x3e, 0x79, 0x19, 0xca, 0xae, 0xba, 0x77, 0xc8, 0xab, - 0xef, 0x3c, 0x7e, 0x33, 0x07, 0x75, 0xc9, 0xc6, 0x24, 0x9b, 0xa0, 0x4c, 0x56, 0x36, 0xa1, 0xca, - 0x86, 0x34, 0x09, 0xee, 0x84, 0x49, 0x9b, 0xea, 0xca, 0x8a, 0xd2, 0x3d, 0x24, 0xd8, 0xe0, 0x17, - 0xd8, 0x9b, 0x9c, 0xe8, 0x17, 0x3c, 0x1a, 0xec, 0x1b, 0xd0, 0x8e, 0x00, 0xad, 0xa7, 0x30, 0x9b, - 0x6a, 0x66, 0x4a, 0xb4, 0x8b, 0xf7, 0x43, 0xff, 0xb7, 0x8b, 0xf7, 0xd1, 0xab, 0xf1, 0x32, 0x83, - 0xac, 0x28, 0xfe, 0xc8, 0xf7, 0x3a, 0xf7, 0x82, 0xc0, 0xda, 0x97, 0x65, 0x08, 0x77, 0x73, 0x3f, - 0xd1, 0xf4, 0x7f, 0xcc, 0x41, 0xed, 0xbd, 0x3e, 0x0e, 0xf6, 0x8f, 0xd2, 0x0f, 0x85, 0x51, 0x61, - 0x3a, 0x16, 0x15, 0x86, 0x4c, 0xbf, 0xa0, 0x30, 0x7d, 0x85, 0x03, 0x2b, 0x2a, 0x1d, 0x98, 0xca, - 0xb6, 0x4b, 0x87, 0xb2, 0xed, 0x72, 0xa6, 0x6d, 0xff, 0xb9, 0x16, 0x89, 0x70, 0x22, 0x6b, 0x4c, - 0x6c, 0xc7, 0x72, 0x87, 0xde, 0x8e, 0x8d, 0x6d, 0x8d, 0xdf, 0x68, 0x50, 0xf9, 0x10, 0xb7, 0xa9, - 0x1f, 0x30, 0xff, 0xa3, 0x20, 0xd3, 0xc6, 0xd8, 0x1a, 0xe7, 0xd2, 0x5b, 0xe3, 0x3b, 0x50, 0x76, - 0x6c, 0xd3, 0x62, 0xfa, 0xc5, 0xc7, 0x1d, 0xb5, 0x25, 0x2b, 0x39, 0x36, 0x57, 0xc4, 0xf1, 0xb3, - 0xfc, 0xbf, 0xab, 0x41, 0x4d, 0xf0, 0x4c, 0x04, 0xe5, 0x9b, 0xb1, 0xe1, 0x34, 0x95, 0xd2, 0xcb, - 0x9f, 0x68, 0xa2, 0x0f, 0xa7, 0x06, 0xc3, 0xde, 0x03, 0x60, 0x42, 0x96, 0xe4, 0xc2, 0x66, 0x96, - 0x94, 0xdc, 0x0a, 0x72, 0x2e, 0xf0, 0x87, 0x53, 0x46, 0x85, 0x51, 0xf1, 0x2e, 0xee, 0x97, 0xa0, - 0xc0, 0xa9, 0xf5, 0xff, 0xd3, 0x60, 0x7e, 0xd5, 0x72, 0xdb, 0x6b, 0x0e, 0xa1, 0x96, 0xd7, 0x9e, - 0x60, 0x13, 0x76, 0x17, 0x4a, 0x7e, 0xcf, 0x74, 0xf1, 0x36, 0x95, 0x2c, 0x5d, 0x1a, 0x31, 0x23, - 0x21, 0x06, 0xa3, 0xe8, 0xf7, 0x1e, 0xe1, 0x6d, 0x8a, 0x7e, 0x16, 0xca, 0x7e, 0xcf, 0x0c, 0x9c, - 0xce, 0x0e, 0x95, 0xd2, 0x1f, 0x83, 0xb8, 0xe4, 0xf7, 0x0c, 0x46, 0x11, 0xcb, 0xad, 0x4c, 0x1f, - 0x32, 0xb7, 0xa2, 0xff, 0xc7, 0xd0, 0xf4, 0x27, 0xb0, 0x81, 0xbb, 0x50, 0x76, 0x3c, 0x6a, 0xda, - 0x0e, 0x09, 0x45, 0x70, 0x5e, 0xad, 0x43, 0x1e, 0xe5, 0x33, 0xe0, 0x6b, 0xea, 0x51, 0x36, 0x36, - 0x7a, 0x1b, 0x60, 0xdb, 0xf5, 0x2d, 0x49, 0x2d, 0x64, 0x70, 0x51, 0x6d, 0x3e, 0x0c, 0x2d, 0xa4, - 0xaf, 0x70, 0x22, 0xd6, 0xc3, 0x60, 0x49, 0xff, 0x4d, 0x83, 0x53, 0x1b, 0x38, 0x10, 0x45, 0x28, - 0x54, 0xe6, 0x39, 0xd7, 0xbd, 0x6d, 0x3f, 0x99, 0x6a, 0xd6, 0x52, 0xa9, 0xe6, 0xef, 0x27, 0xbd, - 0x9a, 0x38, 0x39, 0x89, 0x0b, 0x8f, 0xf0, 0xe4, 0x14, 0x5e, 0xeb, 0x88, 0x93, 0xe7, 0x4c, 0xc6, - 0x32, 0x49, 0x7e, 0xe3, 0x07, 0x70, 0xfd, 0xb7, 0x44, 0x89, 0x85, 0x72, 0x52, 0x2f, 0xae, 0xb0, - 0x8b, 0x20, 0x3d, 0x7d, 0xca, 0xef, 0xff, 0x10, 0x52, 0xbe, 0x23, 0xc3, 0x11, 0xfd, 0xbe, 0x06, - 0x4b, 0xd9, 0x5c, 0x4d, 0x12, 0xa2, 0xdf, 0x86, 0x82, 0xe3, 0x6d, 0xfb, 0x61, 0xda, 0xed, 0x9a, - 0x7a, 0x8b, 0xae, 0x1c, 0x57, 0x10, 0xea, 0x7f, 0x93, 0x83, 0x06, 0x77, 0xea, 0x47, 0xb0, 0xfc, - 0x5d, 0xdc, 0x35, 0x89, 0xf3, 0x29, 0x0e, 0x97, 0xbf, 0x8b, 0xbb, 0x9b, 0xce, 0xa7, 0x38, 0xa1, - 0x19, 0x85, 0xa4, 0x66, 0x24, 0x13, 0x13, 0xc5, 0x11, 0x69, 0xd5, 0x52, 0x32, 0xad, 0xba, 0x08, - 0x45, 0xcf, 0xb7, 0xf1, 0xfa, 0x9a, 0x3c, 0x76, 0xca, 0xbf, 0x81, 0xaa, 0x55, 0x0e, 0xa9, 0x6a, - 0x5f, 0x69, 0xd0, 0x7a, 0x80, 0x69, 0x5a, 0x76, 0x47, 0xa7, 0x65, 0x5f, 0x6b, 0x70, 0x56, 0xc9, - 0xd0, 0x24, 0x0a, 0xf6, 0x66, 0x52, 0xc1, 0xd4, 0x67, 0xc0, 0xa1, 0x21, 0xa5, 0x6e, 0xdd, 0x86, - 0xda, 0x5a, 0xbf, 0xdb, 0x8d, 0xb6, 0x5c, 0x97, 0xa0, 0x16, 0x88, 0x4f, 0x71, 0x44, 0x12, 0xf1, - 0xb7, 0x2a, 0x61, 0xec, 0x20, 0xa4, 0x5f, 0x87, 0xba, 0x24, 0x91, 0x5c, 0xb7, 0xa0, 0x1c, 0xc8, - 0x6f, 0x89, 0x1f, 0xfd, 0xeb, 0xa7, 0x60, 0xde, 0xc0, 0x1d, 0xa6, 0xda, 0xc1, 0x23, 0xc7, 0xdb, - 0x95, 0xc3, 0xe8, 0x9f, 0x6b, 0xb0, 0x90, 0x84, 0xcb, 0xbe, 0x5e, 0x87, 0x92, 0x65, 0xdb, 0x01, - 0x26, 0x64, 0xe4, 0xb2, 0xdc, 0x13, 0x38, 0x46, 0x88, 0x1c, 0x93, 0x5c, 0x6e, 0x6c, 0xc9, 0xe9, - 0x26, 0xcc, 0x3d, 0xc0, 0xf4, 0x31, 0xa6, 0xc1, 0x44, 0x57, 0xf4, 0x4d, 0x76, 0x78, 0xe1, 0xc4, - 0x52, 0x2d, 0xc2, 0x5f, 0xfd, 0x4b, 0x0d, 0x50, 0x7c, 0x84, 0x49, 0x96, 0x39, 0x2e, 0xe5, 0x5c, - 0x52, 0xca, 0xa2, 0x8a, 0xa9, 0xdb, 0xf3, 0x3d, 0xec, 0xd1, 0xf8, 0x76, 0xab, 0x1e, 0x41, 0xb9, - 0xfa, 0x7d, 0xab, 0x01, 0x7a, 0xe4, 0x5b, 0xf6, 0x7d, 0xcb, 0x9d, 0x6c, 0x7b, 0x70, 0x1e, 0x80, - 0x04, 0x6d, 0x53, 0x5a, 0x6b, 0x4e, 0x7a, 0x9f, 0xa0, 0xfd, 0x44, 0x18, 0xec, 0x45, 0xa8, 0xda, - 0x84, 0xca, 0xe6, 0xf0, 0xc6, 0x18, 0x6c, 0x42, 0x45, 0x3b, 0x2f, 0x3f, 0x25, 0xd8, 0x72, 0xb1, - 0x6d, 0xc6, 0x2e, 0xdc, 0xa6, 0x39, 0x5a, 0x43, 0x34, 0x6c, 0x0e, 0xae, 0xdd, 0x9e, 0xc2, 0xe9, - 0xc7, 0x96, 0xd7, 0xb7, 0xdc, 0x55, 0xbf, 0xdb, 0xb3, 0x12, 0x95, 0x8b, 0x69, 0x37, 0xa7, 0x29, - 0xdc, 0xdc, 0x05, 0x51, 0xda, 0x26, 0xb6, 0xd6, 0x9c, 0xd7, 0x69, 0x23, 0x06, 0xd1, 0x09, 0x34, - 0x87, 0xbb, 0x9f, 0x64, 0xa1, 0x38, 0x53, 0x61, 0x57, 0x71, 0xdf, 0x3b, 0x80, 0xe9, 0x6f, 0xc1, - 0x19, 0x5e, 0x66, 0x18, 0x82, 0x12, 0xa9, 0xfd, 0x74, 0x07, 0x9a, 0xa2, 0x83, 0x5f, 0xcb, 0x71, - 0xd7, 0x36, 0xd4, 0xc3, 0x24, 0x8c, 0xdf, 0x4d, 0x66, 0xd4, 0x5f, 0xc9, 0xa8, 0x91, 0x4d, 0x8e, - 0x28, 0xd3, 0xea, 0xcb, 0x30, 0x8b, 0x9f, 0xe3, 0x76, 0x9f, 0x3a, 0x5e, 0x67, 0xc3, 0xb5, 0xbc, - 0x27, 0xbe, 0x0c, 0x28, 0x69, 0x30, 0x7a, 0x05, 0xea, 0x4c, 0xfa, 0x7e, 0x9f, 0x4a, 0x3c, 0x11, - 0x59, 0x92, 0x40, 0xd6, 0x1f, 0x9b, 0xaf, 0x8b, 0x29, 0xb6, 0x25, 0x9e, 0x08, 0x33, 0x69, 0xf0, - 0x90, 0x28, 0x19, 0x98, 0x1c, 0x46, 0x94, 0xff, 0xa5, 0xa5, 0x44, 0x29, 0x7b, 0x38, 0x2a, 0x51, - 0x3e, 0x04, 0xe8, 0xe2, 0xa0, 0x83, 0xd7, 0xb9, 0x53, 0x17, 0x27, 0xf7, 0x65, 0xa5, 0x53, 0x1f, - 0x74, 0xf0, 0x38, 0x24, 0x30, 0x62, 0xb4, 0xfa, 0x03, 0x98, 0x57, 0xa0, 0x30, 0x7f, 0x45, 0xfc, - 0x7e, 0xd0, 0xc6, 0x61, 0xf2, 0x27, 0xfc, 0x65, 0xf1, 0x8d, 0x5a, 0x41, 0x07, 0x53, 0xa9, 0xb4, - 0xf2, 0x4f, 0x7f, 0x9d, 0x5f, 0x42, 0xf1, 0x44, 0x41, 0x42, 0x53, 0x93, 0x37, 0xe6, 0xda, 0xd0, - 0x8d, 0xf9, 0x36, 0xbf, 0xf1, 0x89, 0xd3, 0x4d, 0x58, 0xed, 0xb0, 0xcd, 0xba, 0xc2, 0xb6, 0x7c, - 0x1f, 0x11, 0xfe, 0xb2, 0x5d, 0x72, 0x7d, 0xbd, 0xdb, 0xf3, 0x07, 0x97, 0x1d, 0x63, 0x1f, 0x25, - 0x87, 0x93, 0xc5, 0x39, 0x55, 0xb2, 0xf8, 0x2c, 0x54, 0x02, 0x7f, 0xcf, 0x64, 0xde, 0xcf, 0xe6, - 0x9a, 0x5d, 0x36, 0xca, 0x81, 0xbf, 0xc7, 0x7c, 0xa2, 0x8d, 0x16, 0xa0, 0xb0, 0xed, 0xb8, 0xd1, - 0x81, 0x51, 0xfc, 0xa0, 0x37, 0xd9, 0x19, 0x4a, 0xdc, 0xb8, 0x8e, 0x7d, 0xff, 0x1e, 0x52, 0xe8, - 0x1f, 0xc3, 0x4c, 0x38, 0xa1, 0x09, 0xdf, 0x7c, 0x50, 0x8b, 0xec, 0x86, 0xd5, 0x0c, 0xe2, 0x47, - 0xbf, 0x2e, 0x2e, 0xe2, 0x78, 0xff, 0x89, 0xf5, 0x44, 0x30, 0xcd, 0x30, 0xa4, 0x99, 0xf0, 0x6f, - 0xfd, 0x7f, 0x35, 0x58, 0x4c, 0x63, 0x4f, 0xc2, 0xd2, 0xeb, 0x49, 0xd3, 0x50, 0x97, 0xf5, 0xc7, - 0x47, 0x93, 0x66, 0x21, 0x57, 0xa0, 0xed, 0xf7, 0x3d, 0x2a, 0x7d, 0x0b, 0x5b, 0x81, 0x55, 0xf6, - 0x8f, 0x4e, 0x43, 0xc9, 0xb1, 0x4d, 0x97, 0x1d, 0xb7, 0x44, 0x18, 0x29, 0x3a, 0xf6, 0x23, 0x76, - 0x14, 0x7b, 0x23, 0xdc, 0x1c, 0x8d, 0xbd, 0x04, 0x02, 0xff, 0xda, 0x25, 0x28, 0x87, 0x15, 0x58, - 0xa8, 0x04, 0xf9, 0x7b, 0xae, 0xdb, 0x98, 0x42, 0x35, 0x28, 0xaf, 0xcb, 0x32, 0xa3, 0x86, 0x76, - 0xed, 0xe7, 0x61, 0x36, 0x95, 0xc8, 0x47, 0x65, 0x98, 0x7e, 0xe2, 0x7b, 0xb8, 0x31, 0x85, 0x1a, - 0x50, 0xbb, 0xef, 0x78, 0x56, 0xb0, 0x2f, 0x8e, 0xb9, 0x0d, 0x1b, 0xcd, 0x42, 0x95, 0x1f, 0xf7, - 0x24, 0x00, 0xaf, 0xfc, 0xf3, 0x12, 0xd4, 0x1f, 0x73, 0x76, 0x36, 0x71, 0xf0, 0xcc, 0x69, 0x63, - 0x64, 0x42, 0x23, 0xfd, 0x30, 0x0d, 0xfd, 0x48, 0x6d, 0xfa, 0xea, 0xf7, 0x6b, 0xad, 0x51, 0x4b, - 0xa0, 0x4f, 0xa1, 0x8f, 0x61, 0x26, 0xf9, 0xbc, 0x0b, 0xa9, 0xcf, 0x23, 0xca, 0x37, 0x60, 0x07, - 0x75, 0x6e, 0x42, 0x3d, 0xf1, 0x5a, 0x0b, 0x5d, 0x55, 0xf6, 0xad, 0x7a, 0xd1, 0xd5, 0x52, 0xa7, - 0x08, 0xe2, 0x2f, 0xaa, 0x04, 0xf7, 0xc9, 0x27, 0x15, 0x19, 0xdc, 0x2b, 0xdf, 0x5d, 0x1c, 0xc4, - 0xbd, 0x05, 0x73, 0x43, 0x2f, 0x24, 0xd0, 0x0d, 0x65, 0xff, 0x59, 0x2f, 0x29, 0x0e, 0x1a, 0x62, - 0x0f, 0xd0, 0xf0, 0xab, 0x24, 0x74, 0x53, 0xbd, 0x02, 0x59, 0x6f, 0xb2, 0x5a, 0xb7, 0xc6, 0xc6, - 0x8f, 0x04, 0xf7, 0xab, 0x1a, 0x9c, 0xce, 0x78, 0xd6, 0x80, 0xee, 0x28, 0xbb, 0x1b, 0xfd, 0x36, - 0xa3, 0xf5, 0xea, 0xe1, 0x88, 0x22, 0x46, 0x3c, 0x98, 0x4d, 0x55, 0xfa, 0xa3, 0xeb, 0x99, 0xd5, - 0x8f, 0xc3, 0x4f, 0x1e, 0x5a, 0x3f, 0x1a, 0x0f, 0x39, 0x1a, 0xef, 0x29, 0xcc, 0xa6, 0xca, 0xe3, - 0x33, 0xc6, 0x53, 0x17, 0xd1, 0x1f, 0xb4, 0xa0, 0x1f, 0x41, 0x3d, 0x51, 0xc7, 0x9e, 0xa1, 0xf1, - 0xaa, 0x5a, 0xf7, 0x83, 0xba, 0x7e, 0x0a, 0xb5, 0x78, 0xb9, 0x39, 0x5a, 0xce, 0xb2, 0xa5, 0xa1, - 0x8e, 0x0f, 0x63, 0x4a, 0x83, 0x6a, 0xd2, 0x11, 0xa6, 0x34, 0x54, 0x80, 0x3b, 0xbe, 0x29, 0xc5, - 0xfa, 0x1f, 0x69, 0x4a, 0x87, 0x1e, 0xe2, 0x73, 0x11, 0x95, 0x14, 0xd5, 0xca, 0x68, 0x25, 0x4b, - 0x37, 0xb3, 0xeb, 0xb2, 0x5b, 0x77, 0x0e, 0x45, 0x13, 0x49, 0x71, 0x17, 0x66, 0x92, 0x35, 0xb9, - 0x19, 0x52, 0x54, 0x96, 0x31, 0xb7, 0xae, 0x8f, 0x85, 0x1b, 0x0d, 0xf6, 0x01, 0x54, 0x63, 0x6f, - 0xcd, 0xd1, 0x95, 0x11, 0x7a, 0x1c, 0x7f, 0x78, 0x7d, 0x90, 0x24, 0xdf, 0x83, 0x4a, 0xf4, 0x44, - 0x1c, 0x5d, 0xce, 0xd4, 0xdf, 0xc3, 0x74, 0xb9, 0x09, 0x30, 0x78, 0xff, 0x8d, 0x7e, 0xa8, 0xec, - 0x73, 0xe8, 0x81, 0xf8, 0x41, 0x9d, 0x46, 0xd3, 0x17, 0x95, 0x10, 0xa3, 0xa6, 0x1f, 0x2f, 0xdd, - 0x39, 0xa8, 0xdb, 0x1d, 0xa8, 0x27, 0x0a, 0xee, 0xb2, 0x4c, 0x58, 0x51, 0x07, 0xd9, 0xba, 0x36, - 0x0e, 0x6a, 0xb4, 0x7e, 0x3b, 0x50, 0x4f, 0x94, 0x3f, 0x65, 0x8c, 0xa4, 0xaa, 0xf6, 0xca, 0x18, - 0x49, 0x59, 0x4d, 0xa5, 0x4f, 0xa1, 0xcf, 0x62, 0x95, 0x56, 0x89, 0x6a, 0x36, 0x74, 0x7b, 0x64, - 0x3f, 0xaa, 0x62, 0xbe, 0xd6, 0xca, 0x61, 0x48, 0x22, 0x16, 0xa4, 0x56, 0x09, 0x91, 0x66, 0x6b, - 0xd5, 0x61, 0x56, 0x6a, 0x13, 0x8a, 0xa2, 0xa0, 0x09, 0xe9, 0x19, 0xa5, 0x8b, 0xb1, 0x6a, 0xa7, - 0xd6, 0x0f, 0x94, 0x38, 0xc9, 0x5a, 0x1f, 0xd1, 0xa9, 0x28, 0x58, 0xc9, 0xe8, 0x34, 0x51, 0xcd, - 0x32, 0x6e, 0xa7, 0x06, 0x14, 0xc5, 0x4d, 0x75, 0x46, 0xa7, 0x89, 0x6a, 0x8b, 0xd6, 0x68, 0x1c, - 0x7e, 0x85, 0xa1, 0x4f, 0xa1, 0x0d, 0x28, 0xf0, 0x73, 0x14, 0xba, 0x34, 0xea, 0x12, 0x77, 0x54, - 0x8f, 0x89, 0x7b, 0x5e, 0x7d, 0x0a, 0xfd, 0x22, 0x14, 0x78, 0x56, 0x30, 0xa3, 0xc7, 0xf8, 0x4d, - 0x6c, 0x6b, 0x24, 0x4a, 0xc8, 0xa2, 0x0d, 0xb5, 0xf8, 0xf5, 0x4b, 0x46, 0xc8, 0x52, 0x5c, 0x50, - 0xb5, 0xc6, 0xc1, 0x0c, 0x47, 0x11, 0x66, 0x34, 0x38, 0x53, 0x66, 0x9b, 0xd1, 0xd0, 0x79, 0x35, - 0xdb, 0x8c, 0x86, 0x8f, 0xa8, 0xfa, 0x14, 0xfa, 0x75, 0x0d, 0x9a, 0x59, 0x77, 0x02, 0x28, 0x73, - 0x07, 0x34, 0xea, 0x62, 0xa3, 0xf5, 0xda, 0x21, 0xa9, 0x22, 0x5e, 0x3e, 0x85, 0x79, 0x45, 0xe2, - 0x18, 0xdd, 0xca, 0xea, 0x2f, 0x23, 0xe7, 0xdd, 0xfa, 0xf1, 0xf8, 0x04, 0xd1, 0xd8, 0x1b, 0x50, - 0xe0, 0x09, 0xdf, 0x0c, 0x45, 0x89, 0xe7, 0x8f, 0x33, 0x54, 0x2f, 0x91, 0x2f, 0xd6, 0xa7, 0x10, - 0x86, 0x5a, 0x3c, 0xfb, 0x9b, 0xa1, 0x29, 0x8a, 0xc4, 0x71, 0xeb, 0xea, 0x18, 0x98, 0xd1, 0x30, - 0x26, 0xc0, 0x20, 0xfb, 0x9a, 0x11, 0x87, 0x86, 0x12, 0xc0, 0xad, 0x2b, 0x07, 0xe2, 0xc5, 0x43, - 0x72, 0x2c, 0x9f, 0x9a, 0x11, 0x93, 0x86, 0x33, 0xae, 0x63, 0x9c, 0x13, 0x86, 0x73, 0x7b, 0x19, - 0xe7, 0x84, 0xcc, 0x34, 0x62, 0xeb, 0xd6, 0xd8, 0xf8, 0xd1, 0x7c, 0x3e, 0x81, 0x46, 0x3a, 0x17, - 0x9a, 0x71, 0xfe, 0xcc, 0xc8, 0xc8, 0xb6, 0x6e, 0x8c, 0x89, 0x1d, 0x8f, 0x55, 0x67, 0x87, 0x79, - 0xfa, 0x25, 0x87, 0xee, 0xf0, 0x34, 0xdc, 0x38, 0xb3, 0x8e, 0x67, 0xfc, 0xc6, 0x99, 0x75, 0x22, - 0xbf, 0x27, 0x03, 0x0b, 0xcf, 0x37, 0x64, 0x05, 0x96, 0x78, 0x66, 0x29, 0x23, 0x06, 0x24, 0x93, - 0x35, 0x62, 0x6b, 0x98, 0xcc, 0x9a, 0xa0, 0xec, 0x18, 0x3e, 0x94, 0x88, 0xc9, 0xd8, 0x1a, 0xaa, - 0xd3, 0x30, 0xfa, 0xd4, 0x4a, 0x1f, 0x6a, 0x1b, 0x81, 0xff, 0x7c, 0x3f, 0xcc, 0x23, 0xfc, 0x74, - 0xec, 0xeb, 0xfe, 0x6b, 0xbf, 0x7c, 0xa7, 0xe3, 0xd0, 0x9d, 0xfe, 0x16, 0xd3, 0xe0, 0x5b, 0x02, - 0xf7, 0x86, 0xe3, 0xcb, 0xaf, 0x5b, 0x8e, 0x47, 0x71, 0xe0, 0x59, 0xee, 0x2d, 0xde, 0x97, 0x84, - 0xf6, 0xb6, 0xb6, 0x8a, 0xfc, 0xff, 0xce, 0xff, 0x07, 0x00, 0x00, 0xff, 0xff, 0x84, 0xa2, 0x41, - 0x5e, 0x7d, 0x49, 0x00, 0x00, + // 3800 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x3b, 0xcd, 0x73, 0x1c, 0xc7, + 0x57, 0x9a, 0x5d, 0xed, 0xd7, 0xdb, 0x5d, 0x69, 0xd5, 0x92, 0xe5, 0xf5, 0xfa, 0x4b, 0x9e, 0x5f, + 0xfc, 0xb3, 0x6c, 0xff, 0x6c, 0xc7, 0x72, 0xbe, 0x70, 0x80, 0xc4, 0x96, 0x88, 0xad, 0x8a, 0x6d, + 0x94, 0x51, 0x12, 0x2a, 0xa4, 0x5c, 0x53, 0xa3, 0x9d, 0xd6, 0x6a, 0x4a, 0xb3, 0x33, 0x9b, 0xe9, + 0x5e, 0xcb, 0xca, 0x29, 0x55, 0xa1, 0xa0, 0xa8, 0x84, 0xa4, 0x28, 0x28, 0x3e, 0x0e, 0x70, 0xe0, + 0xe3, 0xc0, 0x01, 0x8a, 0x10, 0x0a, 0x28, 0x2e, 0x70, 0xa0, 0x0a, 0x0e, 0x54, 0xf1, 0x71, 0xe1, + 0xc0, 0x85, 0x7f, 0x80, 0x1b, 0x47, 0x0e, 0x54, 0x7f, 0xcc, 0xec, 0xcc, 0x6c, 0xcf, 0x6a, 0xe5, + 0x8d, 0x91, 0x74, 0x9b, 0x79, 0xfd, 0x5e, 0xf7, 0xeb, 0xd7, 0xef, 0xa3, 0xfb, 0xf5, 0x6b, 0xa8, + 0x75, 0x1d, 0xf7, 0x59, 0x9f, 0xdc, 0xec, 0x05, 0x3e, 0xf5, 0xd1, 0x7c, 0xfc, 0xef, 0xa6, 0xf8, + 0x69, 0xd5, 0xda, 0x7e, 0xb7, 0xeb, 0x7b, 0x02, 0xd8, 0xaa, 0x91, 0xf6, 0x0e, 0xee, 0x5a, 0xe2, + 0x4f, 0xff, 0x7d, 0x0d, 0xd0, 0x6a, 0x80, 0x2d, 0x8a, 0xef, 0xb9, 0x8e, 0x45, 0x0c, 0xfc, 0x59, + 0x1f, 0x13, 0x8a, 0x5e, 0x85, 0xe9, 0x2d, 0x8b, 0xe0, 0xa6, 0xb6, 0xa4, 0x2d, 0x57, 0x57, 0xce, + 0xdd, 0x4c, 0x74, 0x2b, 0xbb, 0x7b, 0x4c, 0x3a, 0xf7, 0x2d, 0x82, 0x0d, 0x8e, 0x89, 0x4e, 0x43, + 0xc9, 0xde, 0x32, 0x3d, 0xab, 0x8b, 0x9b, 0xb9, 0x25, 0x6d, 0xb9, 0x62, 0x14, 0xed, 0xad, 0x27, + 0x56, 0x17, 0xa3, 0x2b, 0x30, 0xdb, 0xf6, 0x5d, 0x17, 0xb7, 0xa9, 0xe3, 0x7b, 0x02, 0x21, 0xcf, + 0x11, 0x66, 0x06, 0x60, 0x8e, 0xb8, 0x00, 0x05, 0x8b, 0xf1, 0xd0, 0x9c, 0xe6, 0xcd, 0xe2, 0x47, + 0x27, 0xd0, 0x58, 0x0b, 0xfc, 0xde, 0xcb, 0xe2, 0x2e, 0x1a, 0x34, 0x1f, 0x1f, 0xf4, 0xf7, 0x34, + 0x98, 0xbb, 0xe7, 0x52, 0x1c, 0x1c, 0x53, 0xa1, 0xfc, 0x4e, 0x0e, 0x4e, 0x8b, 0x55, 0x5b, 0x8d, + 0xd0, 0x8f, 0x92, 0xcb, 0x45, 0x28, 0x0a, 0xad, 0xe2, 0x6c, 0xd6, 0x0c, 0xf9, 0x87, 0xce, 0x03, + 0x90, 0x1d, 0x2b, 0xb0, 0x89, 0xe9, 0xf5, 0xbb, 0xcd, 0xc2, 0x92, 0xb6, 0x5c, 0x30, 0x2a, 0x02, + 0xf2, 0xa4, 0xdf, 0x45, 0x06, 0xcc, 0xb5, 0x7d, 0x8f, 0x38, 0x84, 0x62, 0xaf, 0xbd, 0x6f, 0xba, + 0xf8, 0x19, 0x76, 0x9b, 0xc5, 0x25, 0x6d, 0x79, 0x66, 0xe5, 0xb2, 0x92, 0xef, 0xd5, 0x01, 0xf6, + 0x23, 0x86, 0x6c, 0x34, 0xda, 0x29, 0x88, 0xfe, 0x95, 0x06, 0xa7, 0x98, 0xc2, 0x1c, 0x0b, 0xc1, + 0xe8, 0x7f, 0xa2, 0xc1, 0xc2, 0x43, 0x8b, 0x1c, 0x8f, 0x55, 0x3a, 0x0f, 0x40, 0x9d, 0x2e, 0x36, + 0x09, 0xb5, 0xba, 0x3d, 0xbe, 0x52, 0xd3, 0x46, 0x85, 0x41, 0x36, 0x19, 0x40, 0xff, 0x04, 0x6a, + 0xf7, 0x7d, 0xdf, 0x35, 0x30, 0xe9, 0xf9, 0x1e, 0xc1, 0xe8, 0x0e, 0x14, 0x09, 0xb5, 0x68, 0x9f, + 0x48, 0x26, 0xcf, 0x2a, 0x99, 0xdc, 0xe4, 0x28, 0x86, 0x44, 0x65, 0xfa, 0xfa, 0xcc, 0x72, 0xfb, + 0x82, 0xc7, 0xb2, 0x21, 0x7e, 0xf4, 0x4f, 0x61, 0x66, 0x93, 0x06, 0x8e, 0xd7, 0xf9, 0x01, 0x3b, + 0xaf, 0x84, 0x9d, 0xff, 0xbb, 0x06, 0x67, 0xd6, 0x30, 0x69, 0x07, 0xce, 0xd6, 0x31, 0x31, 0x07, + 0x1d, 0x6a, 0x03, 0xc8, 0xfa, 0x1a, 0x17, 0x75, 0xde, 0x48, 0xc0, 0x52, 0x8b, 0x51, 0x48, 0x2f, + 0xc6, 0x17, 0x05, 0x68, 0xa9, 0x26, 0x35, 0x89, 0xf8, 0x7e, 0x26, 0xb2, 0xd2, 0x1c, 0x27, 0x4a, + 0xd9, 0x98, 0x8c, 0x0b, 0x83, 0xd1, 0x36, 0x39, 0x20, 0x32, 0xe6, 0xf4, 0xac, 0xf2, 0x8a, 0x59, + 0xad, 0xc0, 0xa9, 0x67, 0x4e, 0x40, 0xfb, 0x96, 0x6b, 0xb6, 0x77, 0x2c, 0xcf, 0xc3, 0x2e, 0x97, + 0x13, 0x73, 0x5f, 0xf9, 0xe5, 0x8a, 0x31, 0x2f, 0x1b, 0x57, 0x45, 0x1b, 0x13, 0x16, 0x41, 0xaf, + 0xc1, 0x62, 0x6f, 0x67, 0x9f, 0x38, 0xed, 0x21, 0xa2, 0x02, 0x27, 0x5a, 0x08, 0x5b, 0x13, 0x54, + 0xd7, 0x61, 0xae, 0xcd, 0x3d, 0xa0, 0x6d, 0x32, 0xa9, 0x09, 0x31, 0x16, 0xb9, 0x18, 0x1b, 0xb2, + 0xe1, 0xc3, 0x10, 0xce, 0xd8, 0x0a, 0x91, 0xfb, 0xb4, 0x1d, 0x23, 0x28, 0x71, 0x82, 0x79, 0xd9, + 0xf8, 0x11, 0x6d, 0x0f, 0x68, 0x92, 0xbe, 0xab, 0x9c, 0xf6, 0x5d, 0x4d, 0x28, 0x71, 0x5f, 0x8c, + 0x49, 0xb3, 0xc2, 0xd9, 0x0c, 0x7f, 0xd1, 0x3a, 0xcc, 0x12, 0x6a, 0x05, 0xd4, 0xec, 0xf9, 0xc4, + 0x61, 0x72, 0x21, 0x4d, 0x58, 0xca, 0x2f, 0x57, 0x57, 0x96, 0x94, 0x8b, 0xf4, 0x3e, 0xde, 0x5f, + 0xb3, 0xa8, 0xb5, 0x61, 0x39, 0x81, 0x31, 0xc3, 0x09, 0x37, 0x42, 0x3a, 0xb5, 0x83, 0xac, 0x4e, + 0xe4, 0x20, 0x55, 0x5a, 0x5c, 0x53, 0xfa, 0x2e, 0xe6, 0x49, 0x1f, 0xf9, 0x96, 0x7d, 0x3c, 0x3c, + 0xe9, 0x37, 0x1a, 0x34, 0x0d, 0xec, 0x62, 0x8b, 0x1c, 0x0f, 0x23, 0xd7, 0x7f, 0x53, 0x83, 0x0b, + 0x0f, 0x30, 0x8d, 0x99, 0x0b, 0xb5, 0xa8, 0x43, 0xa8, 0xd3, 0x3e, 0xca, 0x0d, 0x83, 0xfe, 0xad, + 0x06, 0x17, 0x33, 0xd9, 0x9a, 0xc4, 0x7b, 0xbc, 0x09, 0x05, 0xf6, 0x45, 0x9a, 0x39, 0xae, 0xcc, + 0x97, 0xb2, 0x94, 0xf9, 0x63, 0xe6, 0x94, 0xb9, 0x36, 0x0b, 0x7c, 0xfd, 0xbf, 0x34, 0x58, 0xdc, + 0xdc, 0xf1, 0xf7, 0x06, 0x2c, 0xbd, 0x0c, 0x01, 0x25, 0xfd, 0x69, 0x3e, 0xe5, 0x4f, 0xd1, 0x6d, + 0x98, 0xa6, 0xfb, 0x3d, 0xcc, 0x5d, 0xf1, 0xcc, 0xca, 0xf9, 0x9b, 0x8a, 0x7d, 0xf2, 0x4d, 0xc6, + 0xe4, 0x87, 0xfb, 0x3d, 0x6c, 0x70, 0x54, 0x74, 0x15, 0x1a, 0x29, 0x91, 0x87, 0x1e, 0x69, 0x36, + 0x29, 0x73, 0xa2, 0xff, 0x4d, 0x0e, 0x4e, 0x0f, 0x4d, 0x71, 0x12, 0x61, 0xab, 0xc6, 0xce, 0x29, + 0xc7, 0x46, 0x97, 0x21, 0xa6, 0x02, 0xa6, 0x63, 0xb3, 0xad, 0x6c, 0x7e, 0x39, 0x6f, 0xd4, 0x63, + 0x8e, 0xd9, 0x26, 0xe8, 0x06, 0xa0, 0x21, 0x7f, 0x29, 0xdc, 0xf2, 0xb4, 0x31, 0x97, 0x76, 0x98, + 0xdc, 0x29, 0x2b, 0x3d, 0xa6, 0x10, 0xc1, 0xb4, 0xb1, 0xa0, 0x70, 0x99, 0x04, 0xdd, 0x86, 0x05, + 0xc7, 0x7b, 0x8c, 0xbb, 0x7e, 0xb0, 0x6f, 0xf6, 0x70, 0xd0, 0xc6, 0x1e, 0xb5, 0x3a, 0x98, 0x34, + 0x8b, 0x9c, 0xa3, 0xf9, 0xb0, 0x6d, 0x63, 0xd0, 0xa4, 0x7f, 0xaf, 0xc1, 0xa2, 0xd8, 0xca, 0x6e, + 0x58, 0x01, 0x75, 0x8e, 0x3a, 0x74, 0x5f, 0x86, 0x99, 0x5e, 0xc8, 0x87, 0xc0, 0x13, 0x1b, 0xef, + 0x7a, 0x04, 0xe5, 0x56, 0xf6, 0x9d, 0x06, 0x0b, 0x6c, 0x97, 0x79, 0x92, 0x78, 0xfe, 0x73, 0x0d, + 0xe6, 0x1f, 0x5a, 0xe4, 0x24, 0xb1, 0xfc, 0x97, 0x32, 0x04, 0x45, 0x3c, 0x1f, 0xe9, 0x59, 0xec, + 0x0a, 0xcc, 0x26, 0x99, 0x0e, 0xb7, 0x35, 0x33, 0x09, 0xae, 0x89, 0xfe, 0xd7, 0x83, 0x58, 0x75, + 0xc2, 0x38, 0xff, 0x5b, 0x0d, 0xce, 0x3f, 0xc0, 0x34, 0xe2, 0xfa, 0x58, 0xc4, 0xb4, 0x71, 0xb5, + 0xe5, 0x1b, 0x11, 0x91, 0x95, 0xcc, 0x1f, 0x49, 0xe4, 0xfb, 0x2a, 0x07, 0xa7, 0x58, 0x58, 0x38, + 0x1e, 0x4a, 0x30, 0xce, 0xa9, 0x44, 0xa1, 0x28, 0x05, 0x95, 0xa2, 0x44, 0xf1, 0xb4, 0x38, 0x76, + 0x3c, 0xd5, 0xff, 0x22, 0x27, 0xf6, 0x01, 0x71, 0x69, 0x4c, 0xb2, 0x2c, 0x0a, 0x5e, 0x73, 0x4a, + 0x5e, 0x75, 0xa8, 0x45, 0x90, 0xf5, 0xb5, 0x30, 0x3e, 0x26, 0x60, 0xc7, 0x36, 0x3c, 0x7e, 0xad, + 0xc1, 0x62, 0x78, 0x0e, 0xdc, 0xc4, 0x9d, 0x2e, 0xf6, 0xe8, 0x8b, 0xeb, 0x50, 0x5a, 0x03, 0x72, + 0x0a, 0x0d, 0x38, 0x07, 0x15, 0x22, 0xc6, 0x89, 0x8e, 0x78, 0x03, 0x80, 0xfe, 0x77, 0x1a, 0x9c, + 0x1e, 0x62, 0x67, 0x92, 0x45, 0x6c, 0x42, 0xc9, 0xf1, 0x6c, 0xfc, 0x3c, 0xe2, 0x26, 0xfc, 0x65, + 0x2d, 0x5b, 0x7d, 0xc7, 0xb5, 0x23, 0x36, 0xc2, 0x5f, 0x74, 0x09, 0x6a, 0xd8, 0xb3, 0xb6, 0x5c, + 0x6c, 0x72, 0x5c, 0xae, 0xc8, 0x65, 0xa3, 0x2a, 0x60, 0xeb, 0x0c, 0xc4, 0x88, 0xb7, 0x1d, 0xcc, + 0x89, 0x0b, 0x82, 0x58, 0xfe, 0xea, 0xbf, 0xa6, 0xc1, 0x3c, 0xd3, 0x42, 0xc9, 0x3d, 0x79, 0xb9, + 0xd2, 0x5c, 0x82, 0x6a, 0x4c, 0xcd, 0xe4, 0x44, 0xe2, 0x20, 0x7d, 0x17, 0x16, 0x92, 0xec, 0x4c, + 0x22, 0xcd, 0x0b, 0x00, 0xd1, 0x5a, 0x09, 0x6b, 0xc8, 0x1b, 0x31, 0x88, 0xfe, 0x75, 0x2e, 0xcc, + 0xf6, 0x72, 0x31, 0x1d, 0x71, 0x32, 0x8a, 0x2f, 0x49, 0xdc, 0x9f, 0x57, 0x38, 0x84, 0x37, 0xaf, + 0x41, 0x0d, 0x3f, 0xa7, 0x81, 0x65, 0xf6, 0xac, 0xc0, 0xea, 0x0a, 0xb3, 0x1a, 0xcb, 0xf5, 0x56, + 0x39, 0xd9, 0x06, 0xa7, 0x62, 0x83, 0x70, 0x15, 0x11, 0x83, 0x14, 0xc5, 0x20, 0x1c, 0xc2, 0x03, + 0xc6, 0x3f, 0xb1, 0x5d, 0x9c, 0xd4, 0xe6, 0xe3, 0x2e, 0x90, 0xe4, 0x54, 0x0a, 0xe9, 0xa9, 0xfc, + 0xb1, 0x06, 0x0d, 0x3e, 0x05, 0x31, 0x9f, 0x1e, 0xeb, 0x36, 0x45, 0xa3, 0xa5, 0x68, 0x46, 0xd8, + 0xde, 0x4f, 0x41, 0x51, 0xca, 0x3d, 0x3f, 0xae, 0xdc, 0x25, 0xc1, 0x01, 0xd3, 0xd0, 0xff, 0x40, + 0x83, 0x53, 0x29, 0x91, 0x4f, 0xa2, 0xf0, 0x1f, 0x02, 0x12, 0x33, 0xb4, 0x07, 0xd3, 0x0e, 0xe3, + 0xf4, 0x65, 0x65, 0x50, 0x4a, 0x0b, 0xc9, 0x98, 0x73, 0x52, 0x10, 0xa2, 0xff, 0xab, 0x06, 0xe7, + 0x1e, 0x60, 0xca, 0x51, 0xef, 0x33, 0xa7, 0xb3, 0x11, 0xf8, 0x9d, 0x00, 0x13, 0x72, 0x72, 0xf5, + 0xe3, 0xb7, 0xc4, 0xc6, 0x4e, 0x35, 0xa5, 0x49, 0xe4, 0x7f, 0x09, 0x6a, 0x7c, 0x0c, 0x6c, 0x9b, + 0x81, 0xbf, 0x47, 0xa4, 0x1e, 0x55, 0x25, 0xcc, 0xf0, 0xf7, 0xb8, 0x42, 0x50, 0x9f, 0x5a, 0xae, + 0x40, 0x90, 0x11, 0x85, 0x43, 0x58, 0x33, 0xb7, 0xc1, 0x90, 0x31, 0xd6, 0x39, 0x3e, 0xb9, 0x32, + 0xfe, 0x23, 0x0d, 0x4e, 0xa5, 0xa6, 0x32, 0x89, 0x6c, 0x5f, 0x17, 0xdb, 0x4e, 0x31, 0x99, 0x99, + 0x95, 0x8b, 0x4a, 0x9a, 0xd8, 0x60, 0x02, 0x1b, 0x5d, 0x84, 0xea, 0xb6, 0xe5, 0xb8, 0x66, 0x80, + 0x2d, 0xe2, 0x7b, 0x72, 0xa2, 0xc0, 0x40, 0x06, 0x87, 0xe8, 0xff, 0xa0, 0x89, 0x2b, 0xb5, 0x13, + 0xee, 0xf1, 0xfe, 0x30, 0x07, 0xf5, 0x75, 0x8f, 0xe0, 0x80, 0x1e, 0xff, 0xa3, 0x09, 0x7a, 0x07, + 0xaa, 0x7c, 0x62, 0xc4, 0xb4, 0x2d, 0x6a, 0xc9, 0x68, 0x76, 0x41, 0x99, 0x7f, 0x7f, 0x8f, 0xe1, + 0xad, 0x59, 0xd4, 0x32, 0x84, 0x74, 0x08, 0xfb, 0x46, 0x67, 0xa1, 0xb2, 0x63, 0x91, 0x1d, 0x73, + 0x17, 0xef, 0x8b, 0xfd, 0x62, 0xdd, 0x28, 0x33, 0xc0, 0xfb, 0x78, 0x9f, 0xa0, 0x33, 0x50, 0xf6, + 0xfa, 0x5d, 0x61, 0x60, 0xa5, 0x25, 0x6d, 0xb9, 0x6e, 0x94, 0xbc, 0x7e, 0x97, 0x9b, 0xd7, 0x3f, + 0xe7, 0x60, 0xe6, 0x71, 0x9f, 0x1d, 0x84, 0xf8, 0xed, 0x41, 0xdf, 0xa5, 0x2f, 0xa6, 0x8c, 0xd7, + 0x20, 0x2f, 0xb6, 0x14, 0x8c, 0xa2, 0xa9, 0x64, 0x7c, 0x7d, 0x8d, 0x18, 0x0c, 0x89, 0x67, 0xce, + 0xfb, 0xed, 0xb6, 0xdc, 0x9d, 0xe5, 0x39, 0xb3, 0x15, 0x06, 0x11, 0x7b, 0xb3, 0xb3, 0x50, 0xc1, + 0x41, 0x10, 0xed, 0xdd, 0xf8, 0x54, 0x70, 0x10, 0x88, 0x46, 0x1d, 0x6a, 0x56, 0x7b, 0xd7, 0xf3, + 0xf7, 0x5c, 0x6c, 0x77, 0xb0, 0xcd, 0x97, 0xbd, 0x6c, 0x24, 0x60, 0x42, 0x31, 0xd8, 0xc2, 0x9b, + 0x6d, 0x8f, 0xf2, 0xa8, 0x9e, 0x67, 0x8a, 0xc1, 0x20, 0xab, 0x1e, 0x65, 0xcd, 0x36, 0x76, 0x31, + 0xc5, 0xbc, 0xb9, 0x24, 0x9a, 0x05, 0x44, 0x36, 0xf7, 0x7b, 0x11, 0x75, 0x59, 0x34, 0x0b, 0x08, + 0x6b, 0x3e, 0x07, 0x95, 0xc1, 0xf5, 0x40, 0x65, 0x90, 0x46, 0xe4, 0x00, 0xfd, 0x3f, 0x35, 0xa8, + 0xaf, 0xf1, 0xae, 0x4e, 0x80, 0xd2, 0x21, 0x98, 0xc6, 0xcf, 0x7b, 0x81, 0x34, 0x1d, 0xfe, 0x3d, + 0x52, 0x8f, 0xf4, 0x67, 0xd0, 0xd8, 0x70, 0xad, 0x36, 0xde, 0xf1, 0x5d, 0x1b, 0x07, 0x3c, 0xb6, + 0xa3, 0x06, 0xe4, 0xa9, 0xd5, 0x91, 0x9b, 0x07, 0xf6, 0x89, 0xde, 0x92, 0x47, 0x3f, 0xe1, 0x96, + 0x5e, 0x51, 0x46, 0xd9, 0x58, 0x37, 0xb1, 0x8c, 0xea, 0x22, 0x14, 0xf9, 0x95, 0x9d, 0xd8, 0x56, + 0xd4, 0x0c, 0xf9, 0xa7, 0x3f, 0x4d, 0x8c, 0xfb, 0x20, 0xf0, 0xfb, 0x3d, 0xb4, 0x0e, 0xb5, 0xde, + 0x00, 0xc6, 0x74, 0x35, 0x3b, 0xa6, 0xa7, 0x99, 0x36, 0x12, 0xa4, 0xfa, 0x7f, 0xe7, 0xa1, 0xbe, + 0x89, 0xad, 0xa0, 0xbd, 0x73, 0x12, 0x72, 0x30, 0x4c, 0xe2, 0x36, 0x71, 0xe5, 0xaa, 0xb1, 0x4f, + 0x74, 0x1d, 0xe6, 0x62, 0x13, 0x32, 0x3b, 0x4c, 0x40, 0x5c, 0xef, 0x6b, 0x46, 0xa3, 0x97, 0x16, + 0xdc, 0x9b, 0x50, 0xb6, 0x89, 0x6b, 0xf2, 0x25, 0x2a, 0xf1, 0x25, 0x52, 0xcf, 0x6f, 0x8d, 0xb8, + 0x7c, 0x69, 0x4a, 0xb6, 0xf8, 0x40, 0x3f, 0x82, 0xba, 0xdf, 0xa7, 0xbd, 0x3e, 0x35, 0x85, 0xdf, + 0x69, 0x96, 0x39, 0x7b, 0x35, 0x01, 0xe4, 0x6e, 0x89, 0xa0, 0xf7, 0xa0, 0x4e, 0xb8, 0x28, 0xc3, + 0x8d, 0x79, 0x65, 0xdc, 0x0d, 0x62, 0x4d, 0xd0, 0xc9, 0x9d, 0xf9, 0x55, 0x68, 0xd0, 0xc0, 0x7a, + 0x86, 0xdd, 0xd8, 0x65, 0x1c, 0x70, 0x6b, 0x9b, 0x15, 0xf0, 0xc1, 0x45, 0xdc, 0x2d, 0x98, 0xef, + 0xf4, 0xad, 0xc0, 0xf2, 0x28, 0xc6, 0x31, 0xec, 0x2a, 0xc7, 0x46, 0x51, 0x53, 0x44, 0xa0, 0xbf, + 0x0f, 0xd3, 0x0f, 0x1d, 0xca, 0x05, 0xc9, 0x7c, 0x96, 0xc6, 0x8f, 0x41, 0xdc, 0x33, 0x9d, 0x81, + 0x72, 0xe0, 0xef, 0x09, 0x1f, 0x9c, 0xe3, 0x2a, 0x58, 0x0a, 0xfc, 0x3d, 0xee, 0x60, 0x79, 0x09, + 0x83, 0x1f, 0x48, 0xdd, 0xcc, 0x19, 0xf2, 0x4f, 0xff, 0x33, 0x6d, 0xa0, 0x3c, 0xcc, 0x7d, 0x92, + 0x17, 0xf3, 0x9f, 0xef, 0x40, 0x29, 0x10, 0xf4, 0x23, 0x2f, 0x5f, 0xe3, 0x23, 0xf1, 0x18, 0x10, + 0x52, 0x8d, 0x7f, 0x01, 0xf4, 0x4b, 0x1a, 0xd4, 0xde, 0x73, 0xfb, 0xe4, 0x65, 0x28, 0xbb, 0xea, + 0x5a, 0x22, 0xaf, 0xbe, 0x12, 0xf9, 0xf5, 0x1c, 0xd4, 0x25, 0x1b, 0x93, 0x6c, 0x82, 0x32, 0x59, + 0xd9, 0x84, 0x2a, 0x1b, 0xd2, 0x24, 0xb8, 0x13, 0xe6, 0x74, 0xaa, 0x2b, 0x2b, 0x4a, 0xf7, 0x90, + 0x60, 0x83, 0xdf, 0x6f, 0x6f, 0x72, 0xa2, 0x9f, 0xf3, 0x68, 0xb0, 0x6f, 0x40, 0x3b, 0x02, 0xb4, + 0x9e, 0xc2, 0x6c, 0xaa, 0x99, 0x29, 0xd1, 0x2e, 0xde, 0x0f, 0xfd, 0xdf, 0x2e, 0xde, 0x47, 0xaf, + 0xc5, 0xab, 0x10, 0xb2, 0xa2, 0xf8, 0x23, 0xdf, 0xeb, 0xdc, 0x0b, 0x02, 0x6b, 0x5f, 0x56, 0x29, + 0xdc, 0xcd, 0xbd, 0xa5, 0xe9, 0x7f, 0x9f, 0x83, 0xda, 0x07, 0x7d, 0x1c, 0xec, 0x1f, 0xa5, 0x1f, + 0x0a, 0xa3, 0xc2, 0x74, 0x2c, 0x2a, 0x0c, 0x99, 0x7e, 0x41, 0x61, 0xfa, 0x0a, 0x07, 0x56, 0x54, + 0x3a, 0x30, 0x95, 0x6d, 0x97, 0x0e, 0x65, 0xdb, 0xe5, 0x4c, 0xdb, 0xfe, 0x53, 0x2d, 0x12, 0xe1, + 0x44, 0xd6, 0x98, 0xd8, 0x8e, 0xe5, 0x0e, 0xbd, 0x1d, 0x1b, 0xdb, 0x1a, 0xbf, 0xd3, 0xa0, 0xf2, + 0x31, 0x6e, 0x53, 0x3f, 0x60, 0xfe, 0x47, 0x41, 0xa6, 0x8d, 0xb1, 0x35, 0xce, 0xa5, 0xb7, 0xc6, + 0x77, 0xa0, 0xec, 0xd8, 0xa6, 0xc5, 0xf4, 0x8b, 0x8f, 0x3b, 0x6a, 0x4b, 0x56, 0x72, 0x6c, 0xae, + 0x88, 0xe3, 0x5f, 0x02, 0xfc, 0xb6, 0x06, 0x35, 0xc1, 0x33, 0x11, 0x94, 0x6f, 0xc7, 0x86, 0xd3, + 0x54, 0x4a, 0x2f, 0x7f, 0xa2, 0x89, 0x3e, 0x9c, 0x1a, 0x0c, 0x7b, 0x0f, 0x80, 0x09, 0x59, 0x92, + 0x0b, 0x9b, 0x59, 0x52, 0x72, 0x2b, 0xc8, 0xb9, 0xc0, 0x1f, 0x4e, 0x19, 0x15, 0x46, 0xc5, 0xbb, + 0xb8, 0x5f, 0x82, 0x02, 0xa7, 0xd6, 0xff, 0x57, 0x83, 0xf9, 0x55, 0xcb, 0x6d, 0xaf, 0x39, 0x84, + 0x5a, 0x5e, 0x7b, 0x82, 0x4d, 0xd8, 0x5d, 0x28, 0xf9, 0x3d, 0xd3, 0xc5, 0xdb, 0x54, 0xb2, 0x74, + 0x69, 0xc4, 0x8c, 0x84, 0x18, 0x8c, 0xa2, 0xdf, 0x7b, 0x84, 0xb7, 0x29, 0xfa, 0x69, 0x28, 0xfb, + 0x3d, 0x33, 0x70, 0x3a, 0x3b, 0x54, 0x4a, 0x7f, 0x0c, 0xe2, 0x92, 0xdf, 0x33, 0x18, 0x45, 0x2c, + 0xb7, 0x32, 0x7d, 0xc8, 0xdc, 0x8a, 0xfe, 0x6f, 0x43, 0xd3, 0x9f, 0xc0, 0x06, 0xee, 0x42, 0xd9, + 0xf1, 0xa8, 0x69, 0x3b, 0x24, 0x14, 0xc1, 0x79, 0xb5, 0x0e, 0x79, 0x94, 0xcf, 0x80, 0xaf, 0xa9, + 0x47, 0xd9, 0xd8, 0xe8, 0x5d, 0x80, 0x6d, 0xd7, 0xb7, 0x24, 0xb5, 0x90, 0xc1, 0x45, 0xb5, 0xf9, + 0x30, 0xb4, 0x90, 0xbe, 0xc2, 0x89, 0x58, 0x0f, 0x83, 0x25, 0xfd, 0x17, 0x0d, 0x4e, 0x6d, 0xe0, + 0x40, 0xd4, 0xa8, 0x50, 0x99, 0x06, 0x5d, 0xf7, 0xb6, 0xfd, 0x64, 0x26, 0x5a, 0x4b, 0x65, 0xa2, + 0x7f, 0x98, 0xec, 0x6b, 0xe2, 0xe4, 0x24, 0xee, 0x43, 0xc2, 0x93, 0x53, 0x78, 0xeb, 0x23, 0x4e, + 0x9e, 0x33, 0x19, 0xcb, 0x24, 0xf9, 0x8d, 0x1f, 0xc0, 0xf5, 0xdf, 0x10, 0x15, 0x18, 0xca, 0x49, + 0xbd, 0xb8, 0xc2, 0x2e, 0x82, 0xf4, 0xf4, 0x29, 0xbf, 0xff, 0x63, 0x48, 0xf9, 0x8e, 0x0c, 0x47, + 0xf4, 0xbb, 0x1a, 0x2c, 0x65, 0x73, 0x35, 0x49, 0x88, 0x7e, 0x17, 0x0a, 0x8e, 0xb7, 0xed, 0x87, + 0x69, 0xb7, 0x6b, 0xea, 0x2d, 0xba, 0x72, 0x5c, 0x41, 0xa8, 0xff, 0x55, 0x0e, 0x1a, 0xdc, 0xa9, + 0x1f, 0xc1, 0xf2, 0x77, 0x71, 0xd7, 0x24, 0xce, 0xe7, 0x38, 0x5c, 0xfe, 0x2e, 0xee, 0x6e, 0x3a, + 0x9f, 0xe3, 0x84, 0x66, 0x14, 0x92, 0x9a, 0x31, 0x3a, 0xab, 0x1c, 0x4f, 0xab, 0x96, 0x92, 0x69, + 0xd5, 0x45, 0x28, 0x7a, 0xbe, 0x8d, 0xd7, 0xd7, 0xe4, 0xb1, 0x53, 0xfe, 0x0d, 0x54, 0xad, 0x72, + 0x48, 0x55, 0xfb, 0x46, 0x83, 0xd6, 0x03, 0x4c, 0xd3, 0xb2, 0x3b, 0x3a, 0x2d, 0xfb, 0x56, 0x83, + 0xb3, 0x4a, 0x86, 0x26, 0x51, 0xb0, 0xb7, 0x93, 0x0a, 0xa6, 0x3e, 0x03, 0x0e, 0x0d, 0x29, 0x75, + 0xeb, 0x36, 0xd4, 0xd6, 0xfa, 0xdd, 0x6e, 0xb4, 0xe5, 0xba, 0x04, 0xb5, 0x40, 0x7c, 0x8a, 0x23, + 0x92, 0x88, 0xbf, 0x55, 0x09, 0x63, 0x07, 0x21, 0xfd, 0x3a, 0xd4, 0x25, 0x89, 0xe4, 0xba, 0x05, + 0xe5, 0x40, 0x7e, 0x4b, 0xfc, 0xe8, 0x5f, 0x3f, 0x05, 0xf3, 0x06, 0xee, 0x30, 0xd5, 0x0e, 0x1e, + 0x39, 0xde, 0xae, 0x1c, 0x46, 0xff, 0x52, 0x83, 0x85, 0x24, 0x5c, 0xf6, 0xf5, 0x06, 0x94, 0x2c, + 0xdb, 0x0e, 0x30, 0x21, 0x23, 0x97, 0xe5, 0x9e, 0xc0, 0x31, 0x42, 0xe4, 0x98, 0xe4, 0x72, 0x63, + 0x4b, 0x4e, 0x37, 0x61, 0xee, 0x01, 0xa6, 0x8f, 0x31, 0x0d, 0x26, 0xba, 0xc1, 0x6f, 0xb2, 0xc3, + 0x0b, 0x27, 0x96, 0x6a, 0x11, 0xfe, 0xea, 0x5f, 0x6b, 0x80, 0xe2, 0x23, 0x4c, 0xb2, 0xcc, 0x71, + 0x29, 0xe7, 0x92, 0x52, 0x16, 0x45, 0x4e, 0xdd, 0x9e, 0xef, 0x61, 0x8f, 0xc6, 0xb7, 0x5b, 0xf5, + 0x08, 0xca, 0xd5, 0xef, 0x7b, 0x0d, 0xd0, 0x23, 0xdf, 0xb2, 0xef, 0x5b, 0xee, 0x64, 0xdb, 0x83, + 0xf3, 0x00, 0x24, 0x68, 0x9b, 0xd2, 0x5a, 0x73, 0xd2, 0xfb, 0x04, 0xed, 0x27, 0xc2, 0x60, 0x2f, + 0x42, 0xd5, 0x26, 0x54, 0x36, 0x87, 0x17, 0xca, 0x60, 0x13, 0x2a, 0xda, 0x79, 0x75, 0x2a, 0xc1, + 0x96, 0x8b, 0x6d, 0x33, 0x76, 0x1f, 0x37, 0xcd, 0xd1, 0x1a, 0xa2, 0x61, 0x73, 0x70, 0x2b, 0xf7, + 0x14, 0x4e, 0x3f, 0xb6, 0xbc, 0xbe, 0xe5, 0xae, 0xfa, 0xdd, 0x9e, 0x95, 0x28, 0x6c, 0x4c, 0xbb, + 0x39, 0x4d, 0xe1, 0xe6, 0x2e, 0x88, 0xca, 0x37, 0xb1, 0xb5, 0xe6, 0xbc, 0x4e, 0x1b, 0x31, 0x88, + 0x4e, 0xa0, 0x39, 0xdc, 0xfd, 0x24, 0x0b, 0xc5, 0x99, 0x0a, 0xbb, 0x8a, 0xfb, 0xde, 0x01, 0x4c, + 0x7f, 0x07, 0xce, 0xf0, 0x2a, 0xc4, 0x10, 0x94, 0x48, 0xed, 0xa7, 0x3b, 0xd0, 0x14, 0x1d, 0xfc, + 0x4a, 0x8e, 0xbb, 0xb6, 0xa1, 0x1e, 0x26, 0x61, 0xfc, 0x6e, 0x32, 0xa3, 0xfe, 0x4a, 0x46, 0x09, + 0x6d, 0x72, 0x44, 0x99, 0x56, 0x5f, 0x86, 0x59, 0xfc, 0x1c, 0xb7, 0xfb, 0xd4, 0xf1, 0x3a, 0x1b, + 0xae, 0xe5, 0x3d, 0xf1, 0x65, 0x40, 0x49, 0x83, 0xd1, 0x2b, 0x50, 0x67, 0xd2, 0xf7, 0xfb, 0x54, + 0xe2, 0x89, 0xc8, 0x92, 0x04, 0xb2, 0xfe, 0xd8, 0x7c, 0x5d, 0x4c, 0xb1, 0x2d, 0xf1, 0x44, 0x98, + 0x49, 0x83, 0x87, 0x44, 0xc9, 0xc0, 0xe4, 0x30, 0xa2, 0xfc, 0x0f, 0x2d, 0x25, 0x4a, 0xd9, 0xc3, + 0x51, 0x89, 0xf2, 0x21, 0x40, 0x17, 0x07, 0x1d, 0xbc, 0xce, 0x9d, 0xba, 0x38, 0xb9, 0x2f, 0x2b, + 0x9d, 0xfa, 0xa0, 0x83, 0xc7, 0x21, 0x81, 0x11, 0xa3, 0xd5, 0x1f, 0xc0, 0xbc, 0x02, 0x85, 0xf9, + 0x2b, 0xe2, 0xf7, 0x83, 0x36, 0x0e, 0x93, 0x3f, 0xe1, 0x2f, 0x8b, 0x6f, 0xd4, 0x0a, 0x3a, 0x98, + 0x4a, 0xa5, 0x95, 0x7f, 0xfa, 0x1b, 0xfc, 0x12, 0x8a, 0x27, 0x0a, 0x12, 0x9a, 0x9a, 0xbc, 0x50, + 0xd7, 0x86, 0x2e, 0xd4, 0xb7, 0xf9, 0x8d, 0x4f, 0x9c, 0x6e, 0xc2, 0x62, 0x88, 0x6d, 0xd6, 0x15, + 0xb6, 0xe5, 0xf3, 0x89, 0xf0, 0x97, 0xed, 0x92, 0xeb, 0xeb, 0xdd, 0x9e, 0x3f, 0xb8, 0xec, 0x18, + 0xfb, 0x28, 0x39, 0x9c, 0x2c, 0xce, 0xa9, 0x92, 0xc5, 0x67, 0xa1, 0x12, 0xf8, 0x7b, 0x26, 0xf3, + 0x7e, 0x36, 0xd7, 0xec, 0xb2, 0x51, 0x0e, 0xfc, 0x3d, 0xe6, 0x13, 0x6d, 0xb4, 0x00, 0x85, 0x6d, + 0xc7, 0x8d, 0x0e, 0x8c, 0xe2, 0x07, 0xbd, 0xcd, 0xce, 0x50, 0xe2, 0xc6, 0x75, 0xec, 0xeb, 0xf9, + 0x90, 0x42, 0xff, 0x14, 0x66, 0xc2, 0x09, 0x4d, 0xf8, 0x24, 0x84, 0x5a, 0x64, 0x37, 0x2c, 0x76, + 0x10, 0x3f, 0xfa, 0x75, 0x71, 0x11, 0xc7, 0xfb, 0x4f, 0xac, 0x27, 0x82, 0x69, 0x86, 0x21, 0xcd, + 0x84, 0x7f, 0xeb, 0xff, 0xa3, 0xc1, 0x62, 0x1a, 0x7b, 0x12, 0x96, 0xde, 0x48, 0x9a, 0x86, 0xba, + 0xea, 0x3f, 0x3e, 0x9a, 0x34, 0x0b, 0xb9, 0x02, 0x6d, 0xbf, 0xef, 0x51, 0xe9, 0x5b, 0xd8, 0x0a, + 0xac, 0xb2, 0x7f, 0x74, 0x1a, 0x4a, 0x8e, 0x6d, 0xba, 0xec, 0xb8, 0x25, 0xc2, 0x48, 0xd1, 0xb1, + 0x1f, 0xb1, 0xa3, 0xd8, 0x9b, 0xe1, 0xe6, 0x68, 0xec, 0x25, 0x10, 0xf8, 0xd7, 0x2e, 0x41, 0x39, + 0x2c, 0xd0, 0x42, 0x25, 0xc8, 0xdf, 0x73, 0xdd, 0xc6, 0x14, 0xaa, 0x41, 0x79, 0x5d, 0x56, 0x21, + 0x35, 0xb4, 0x6b, 0x3f, 0x0b, 0xb3, 0xa9, 0x44, 0x3e, 0x2a, 0xc3, 0xf4, 0x13, 0xdf, 0xc3, 0x8d, + 0x29, 0xd4, 0x80, 0xda, 0x7d, 0xc7, 0xb3, 0x82, 0x7d, 0x71, 0xcc, 0x6d, 0xd8, 0x68, 0x16, 0xaa, + 0xfc, 0xb8, 0x27, 0x01, 0x78, 0xe5, 0x1f, 0x97, 0xa0, 0xfe, 0x98, 0xb3, 0xb3, 0x89, 0x83, 0x67, + 0x4e, 0x1b, 0x23, 0x13, 0x1a, 0xe9, 0x77, 0x6b, 0xe8, 0x27, 0x6a, 0xd3, 0x57, 0x3f, 0x6f, 0x6b, + 0x8d, 0x5a, 0x02, 0x7d, 0x0a, 0x7d, 0x0a, 0x33, 0xc9, 0xd7, 0x5f, 0x48, 0x7d, 0x1e, 0x51, 0x3e, + 0x11, 0x3b, 0xa8, 0x73, 0x13, 0xea, 0x89, 0xc7, 0x5c, 0xe8, 0xaa, 0xb2, 0x6f, 0xd5, 0x83, 0xaf, + 0x96, 0x3a, 0x45, 0x10, 0x7f, 0x70, 0x25, 0xb8, 0x4f, 0xbe, 0xb8, 0xc8, 0xe0, 0x5e, 0xf9, 0x2c, + 0xe3, 0x20, 0xee, 0x2d, 0x98, 0x1b, 0x7a, 0x40, 0x81, 0x6e, 0x28, 0xfb, 0xcf, 0x7a, 0x68, 0x71, + 0xd0, 0x10, 0x7b, 0x80, 0x86, 0x1f, 0x2d, 0xa1, 0x9b, 0xea, 0x15, 0xc8, 0x7a, 0xb2, 0xd5, 0xba, + 0x35, 0x36, 0x7e, 0x24, 0xb8, 0x5f, 0xd6, 0xe0, 0x74, 0xc6, 0xab, 0x07, 0x74, 0x47, 0xd9, 0xdd, + 0xe8, 0xa7, 0x1b, 0xad, 0xd7, 0x0e, 0x47, 0x14, 0x31, 0xe2, 0xc1, 0x6c, 0xea, 0x21, 0x00, 0xba, + 0x9e, 0x59, 0x1c, 0x39, 0xfc, 0x22, 0xa2, 0xf5, 0x93, 0xf1, 0x90, 0xa3, 0xf1, 0x9e, 0xc2, 0x6c, + 0xaa, 0x7a, 0x3e, 0x63, 0x3c, 0x75, 0x8d, 0xfd, 0x41, 0x0b, 0xfa, 0x09, 0xd4, 0x13, 0x65, 0xee, + 0x19, 0x1a, 0xaf, 0x2a, 0x85, 0x3f, 0xa8, 0xeb, 0xa7, 0x50, 0x8b, 0x57, 0xa3, 0xa3, 0xe5, 0x2c, + 0x5b, 0x1a, 0xea, 0xf8, 0x30, 0xa6, 0x34, 0x28, 0x36, 0x1d, 0x61, 0x4a, 0x43, 0xf5, 0xb9, 0xe3, + 0x9b, 0x52, 0xac, 0xff, 0x91, 0xa6, 0x74, 0xe8, 0x21, 0xbe, 0x14, 0x51, 0x49, 0x51, 0xcc, 0x8c, + 0x56, 0xb2, 0x74, 0x33, 0xbb, 0x6c, 0xbb, 0x75, 0xe7, 0x50, 0x34, 0x91, 0x14, 0x77, 0x61, 0x26, + 0x59, 0xb2, 0x9b, 0x21, 0x45, 0x65, 0x95, 0x73, 0xeb, 0xfa, 0x58, 0xb8, 0xd1, 0x60, 0x1f, 0x41, + 0x35, 0xf6, 0x14, 0x1d, 0x5d, 0x19, 0xa1, 0xc7, 0xf1, 0x77, 0xd9, 0x07, 0x49, 0xf2, 0x03, 0xa8, + 0x44, 0x2f, 0xc8, 0xd1, 0xe5, 0x4c, 0xfd, 0x3d, 0x4c, 0x97, 0x9b, 0x00, 0x83, 0xe7, 0xe1, 0xe8, + 0xc7, 0xca, 0x3e, 0x87, 0xde, 0x8f, 0x1f, 0xd4, 0x69, 0x34, 0x7d, 0x51, 0x09, 0x31, 0x6a, 0xfa, + 0xf1, 0xd2, 0x9d, 0x83, 0xba, 0xdd, 0x81, 0x7a, 0xa2, 0xe0, 0x2e, 0xcb, 0x84, 0x15, 0x75, 0x90, + 0xad, 0x6b, 0xe3, 0xa0, 0x46, 0xeb, 0xb7, 0x03, 0xf5, 0x44, 0xf9, 0x53, 0xc6, 0x48, 0xaa, 0x6a, + 0xaf, 0x8c, 0x91, 0x94, 0xd5, 0x54, 0xfa, 0x14, 0xfa, 0x22, 0x56, 0x69, 0x95, 0xa8, 0x66, 0x43, + 0xb7, 0x47, 0xf6, 0xa3, 0x2a, 0xe6, 0x6b, 0xad, 0x1c, 0x86, 0x24, 0x62, 0x41, 0x6a, 0x95, 0x10, + 0x69, 0xb6, 0x56, 0x1d, 0x66, 0xa5, 0x36, 0xa1, 0x28, 0x0a, 0x9a, 0x90, 0x9e, 0x51, 0xba, 0x18, + 0xab, 0x76, 0x6a, 0xfd, 0x48, 0x89, 0x93, 0xac, 0xf5, 0x11, 0x9d, 0x8a, 0x82, 0x95, 0x8c, 0x4e, + 0x13, 0xd5, 0x2c, 0xe3, 0x76, 0x6a, 0x40, 0x51, 0xdc, 0x54, 0x67, 0x74, 0x9a, 0xa8, 0xb6, 0x68, + 0x8d, 0xc6, 0xe1, 0x57, 0x18, 0xfa, 0x14, 0xda, 0x80, 0x02, 0x3f, 0x47, 0xa1, 0x4b, 0xa3, 0x2e, + 0x71, 0x47, 0xf5, 0x98, 0xb8, 0xe7, 0xd5, 0xa7, 0xd0, 0xcf, 0x43, 0x81, 0x67, 0x05, 0x33, 0x7a, + 0x8c, 0xdf, 0xc4, 0xb6, 0x46, 0xa2, 0x84, 0x2c, 0xda, 0x50, 0x8b, 0x5f, 0xbf, 0x64, 0x84, 0x2c, + 0xc5, 0x05, 0x55, 0x6b, 0x1c, 0xcc, 0x70, 0x14, 0x61, 0x46, 0x83, 0x33, 0x65, 0xb6, 0x19, 0x0d, + 0x9d, 0x57, 0xb3, 0xcd, 0x68, 0xf8, 0x88, 0xaa, 0x4f, 0xa1, 0x5f, 0xd5, 0xa0, 0x99, 0x75, 0x27, + 0x80, 0x32, 0x77, 0x40, 0xa3, 0x2e, 0x36, 0x5a, 0xaf, 0x1f, 0x92, 0x2a, 0xe2, 0xe5, 0x73, 0x98, + 0x57, 0x24, 0x8e, 0xd1, 0xad, 0xac, 0xfe, 0x32, 0x72, 0xde, 0xad, 0x57, 0xc7, 0x27, 0x88, 0xc6, + 0xde, 0x80, 0x02, 0x4f, 0xf8, 0x66, 0x28, 0x4a, 0x3c, 0x7f, 0x9c, 0xa1, 0x7a, 0x89, 0x7c, 0xb1, + 0x3e, 0x85, 0x30, 0xd4, 0xe2, 0xd9, 0xdf, 0x0c, 0x4d, 0x51, 0x24, 0x8e, 0x5b, 0x57, 0xc7, 0xc0, + 0x8c, 0x86, 0x31, 0x01, 0x06, 0xd9, 0xd7, 0x8c, 0x38, 0x34, 0x94, 0x00, 0x6e, 0x5d, 0x39, 0x10, + 0x2f, 0x1e, 0x92, 0x63, 0xf9, 0xd4, 0x8c, 0x98, 0x34, 0x9c, 0x71, 0x1d, 0xe3, 0x9c, 0x30, 0x9c, + 0xdb, 0xcb, 0x38, 0x27, 0x64, 0xa6, 0x11, 0x5b, 0xb7, 0xc6, 0xc6, 0x8f, 0xe6, 0xf3, 0x19, 0x34, + 0xd2, 0xb9, 0xd0, 0x8c, 0xf3, 0x67, 0x46, 0x46, 0xb6, 0x75, 0x63, 0x4c, 0xec, 0x78, 0xac, 0x3a, + 0x3b, 0xcc, 0xd3, 0x2f, 0x38, 0x74, 0x87, 0xa7, 0xe1, 0xc6, 0x99, 0x75, 0x3c, 0xe3, 0x37, 0xce, + 0xac, 0x13, 0xf9, 0x3d, 0x19, 0x58, 0x78, 0xbe, 0x21, 0x2b, 0xb0, 0xc4, 0x33, 0x4b, 0x19, 0x31, + 0x20, 0x99, 0xac, 0x11, 0x5b, 0xc3, 0x64, 0xd6, 0x04, 0x65, 0xc7, 0xf0, 0xa1, 0x44, 0x4c, 0xc6, + 0xd6, 0x50, 0x9d, 0x86, 0xd1, 0xa7, 0x56, 0xfa, 0x50, 0xdb, 0x08, 0xfc, 0xe7, 0xfb, 0x61, 0x1e, + 0xe1, 0xff, 0xc7, 0xbe, 0xee, 0xbf, 0xfe, 0x8b, 0x77, 0x3a, 0x0e, 0xdd, 0xe9, 0x6f, 0x31, 0x0d, + 0xbe, 0x25, 0x70, 0x6f, 0x38, 0xbe, 0xfc, 0xba, 0xe5, 0x78, 0x14, 0x07, 0x9e, 0xe5, 0xde, 0xe2, + 0x7d, 0x49, 0x68, 0x6f, 0x6b, 0xab, 0xc8, 0xff, 0xef, 0xfc, 0x5f, 0x00, 0x00, 0x00, 0xff, 0xff, + 0xa8, 0x9d, 0xf6, 0x02, 0x9c, 0x49, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. diff --git a/internal/proxy/task.go b/internal/proxy/task.go index a60ade6b4f81d96c0e32d1fa85dc6072073b58e5..db7fde30eabc084bb807807560b047474ef9385b 100644 --- a/internal/proxy/task.go +++ b/internal/proxy/task.go @@ -3213,6 +3213,12 @@ func (cit *createIndexTask) PreExecute(ctx context.Context) error { collName, fieldName := cit.CollectionName, cit.FieldName + collID, err := globalMetaCache.GetCollectionID(ctx, collName) + if err != nil { + return err + } + cit.collectionID = collID + if err := validateCollectionName(collName); err != nil { return err } @@ -3245,6 +3251,18 @@ func (cit *createIndexTask) PreExecute(ctx context.Context) error { indexType = indexparamcheck.IndexFaissIvfPQ // IVF_PQ is the default index type } + // skip params check of non-vector field. + vecDataTypes := []schemapb.DataType{ + schemapb.DataType_FloatVector, + schemapb.DataType_BinaryVector, + } + schema, _ := globalMetaCache.GetCollectionSchema(ctx, collName) + for _, f := range schema.GetFields() { + if f.GetName() == fieldName && !funcutil.SliceContain(vecDataTypes, f.GetDataType()) { + return indexparamcheck.CheckIndexValid(f.GetDataType(), indexType, indexParams) + } + } + adapter, err := indexparamcheck.GetConfAdapterMgrInstance().GetAdapter(indexType) if err != nil { log.Warn("Failed to get conf adapter", zap.String("index_type", indexType)) @@ -3257,8 +3275,6 @@ func (cit *createIndexTask) PreExecute(ctx context.Context) error { return fmt.Errorf("invalid index params: %v", cit.CreateIndexRequest.ExtraParams) } - collID, _ := globalMetaCache.GetCollectionID(ctx, collName) - cit.collectionID = collID return nil } diff --git a/internal/querycoord/cluster_test.go b/internal/querycoord/cluster_test.go index ff7a62296fcb7091b0d215e36aeaf7798bd968a7..659cd8ed366aaa800006302ef1aae3e8cc3093d0 100644 --- a/internal/querycoord/cluster_test.go +++ b/internal/querycoord/cluster_test.go @@ -25,10 +25,11 @@ import ( "strconv" "testing" + "github.com/milvus-io/milvus/internal/util/indexcgowrapper" + "github.com/stretchr/testify/assert" "go.uber.org/zap" - "github.com/milvus-io/milvus/internal/indexnode" "github.com/milvus-io/milvus/internal/kv" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/log" @@ -311,12 +312,12 @@ func generateIndex(indexBuildID UniqueID, cm storage.ChunkManager) ([]string, er } } - index, err := indexnode.NewCIndex(typeParams, indexParams) + index, err := indexcgowrapper.NewCgoIndex(schemapb.DataType_FloatVector, typeParams, indexParams) if err != nil { return nil, err } - err = index.BuildFloatVecIndexWithoutIds(indexRowData) + err = index.Build(indexcgowrapper.GenFloatVecDataset(indexRowData)) if err != nil { return nil, err } diff --git a/internal/querynode/mock_test.go b/internal/querynode/mock_test.go index 7687eb570da74d1a0df2a2d7cfd6c29626fd3c42..eb1cc4c6a45287db25e402a2cefbe48c5aeef0e7 100644 --- a/internal/querynode/mock_test.go +++ b/internal/querynode/mock_test.go @@ -24,11 +24,12 @@ import ( "math/rand" "strconv" + "github.com/milvus-io/milvus/internal/util/indexcgowrapper" + "github.com/golang/protobuf/proto" "go.uber.org/zap" "github.com/milvus-io/milvus/internal/common" - "github.com/milvus-io/milvus/internal/indexnode" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/log" "github.com/milvus-io/milvus/internal/mq/msgstream" @@ -242,12 +243,12 @@ func genIndexBinarySet() ([][]byte, error) { } } - index, err := indexnode.NewCIndex(typeParams, indexParams) + index, err := indexcgowrapper.NewCgoIndex(schemapb.DataType_FloatVector, typeParams, indexParams) if err != nil { return nil, err } - err = index.BuildFloatVecIndexWithoutIds(indexRowData) + err = index.Build(indexcgowrapper.GenFloatVecDataset(indexRowData)) if err != nil { return nil, err } @@ -348,12 +349,12 @@ func generateIndex(segmentID UniqueID) ([]string, error) { } } - index, err := indexnode.NewCIndex(typeParams, indexParams) + index, err := indexcgowrapper.NewCgoIndex(schemapb.DataType_FloatVector, typeParams, indexParams) if err != nil { return nil, err } - err = index.BuildFloatVecIndexWithoutIds(indexRowData) + err = index.Build(indexcgowrapper.GenFloatVecDataset(indexRowData)) if err != nil { return nil, err } @@ -415,12 +416,12 @@ func generateAndSaveIndex(segmentID UniqueID, msgLength int, indexType, metricTy } } - index, err := indexnode.NewCIndex(typeParams, indexParams) + index, err := indexcgowrapper.NewCgoIndex(schemapb.DataType_FloatVector, typeParams, indexParams) if err != nil { return nil, err } - err = index.BuildFloatVecIndexWithoutIds(indexRowData) + err = index.Build(indexcgowrapper.GenFloatVecDataset(indexRowData)) if err != nil { return nil, err } diff --git a/internal/rootcoord/task.go b/internal/rootcoord/task.go index 8760ba403ba52ace702521dbe5236d0d27f6512b..826ada02a9261b04e41edc147e7e36ccb0b8cd87 100644 --- a/internal/rootcoord/task.go +++ b/internal/rootcoord/task.go @@ -845,7 +845,10 @@ func (t *CreateIndexReqTask) Execute(ctx context.Context) error { if t.Type() != commonpb.MsgType_CreateIndex { return fmt.Errorf("create index, msg type = %s", commonpb.MsgType_name[int32(t.Type())]) } - indexName := Params.CommonCfg.DefaultIndexName //TODO, get name from request + indexName := t.Req.GetIndexName() + if len(indexName) <= 0 { + indexName = Params.CommonCfg.DefaultIndexName //TODO, get name from request + } indexID, _, err := t.core.IDAllocator(1) log.Debug("RootCoord CreateIndexReqTask", zap.Any("indexID", indexID), zap.Error(err)) if err != nil { @@ -856,6 +859,12 @@ func (t *CreateIndexReqTask) Execute(ctx context.Context) error { IndexID: indexID, IndexParams: t.Req.ExtraParams, } + log.Info("create index for collection", + zap.String("collection", t.Req.GetCollectionName()), + zap.String("field", t.Req.GetFieldName()), + zap.String("index", indexName), + zap.Int64("index_id", indexID), + zap.Any("params", t.Req.GetExtraParams())) collMeta, err := t.core.MetaTable.GetCollectionByName(t.Req.CollectionName, 0) if err != nil { return err @@ -875,9 +884,6 @@ func (t *CreateIndexReqTask) Execute(ctx context.Context) error { log.Debug("RootCoord CreateIndexReqTask metaTable.GetNotIndexedSegments", zap.Error(err)) return err } - if field.DataType != schemapb.DataType_FloatVector && field.DataType != schemapb.DataType_BinaryVector { - return fmt.Errorf("field name = %s, data type = %s", t.Req.FieldName, schemapb.DataType_name[int32(field.DataType)]) - } collectionID := collMeta.ID cnt := 0 diff --git a/internal/util/indexcgowrapper/codec_index_test.go b/internal/util/indexcgowrapper/codec_index_test.go new file mode 100644 index 0000000000000000000000000000000000000000..d32236c863d142f644df72edd1bee53ee5e8884e --- /dev/null +++ b/internal/util/indexcgowrapper/codec_index_test.go @@ -0,0 +1,312 @@ +//go:build linux +// +build linux + +package indexcgowrapper + +import ( + "math/rand" + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/milvus-io/milvus/internal/util/funcutil" + + "github.com/milvus-io/milvus/internal/storage" + + "github.com/milvus-io/milvus/internal/proto/schemapb" +) + +type indexTestCase struct { + dtype schemapb.DataType + typeParams map[string]string + indexParams map[string]string +} + +func generateBoolArray(numRows int) []bool { + ret := make([]bool, 0, numRows) + for i := 0; i < numRows; i++ { + ret = append(ret, rand.Int()%2 == 0) + } + return ret +} + +func generateInt8Array(numRows int) []int8 { + ret := make([]int8, 0, numRows) + for i := 0; i < numRows; i++ { + ret = append(ret, int8(rand.Int())) + } + return ret +} + +func generateInt16Array(numRows int) []int16 { + ret := make([]int16, 0, numRows) + for i := 0; i < numRows; i++ { + ret = append(ret, int16(rand.Int())) + } + return ret +} + +func generateInt32Array(numRows int) []int32 { + ret := make([]int32, 0, numRows) + for i := 0; i < numRows; i++ { + ret = append(ret, int32(rand.Int())) + } + return ret +} + +func generateInt64Array(numRows int) []int64 { + ret := make([]int64, 0, numRows) + for i := 0; i < numRows; i++ { + ret = append(ret, int64(rand.Int())) + } + return ret +} + +func generateFloat32Array(numRows int) []float32 { + ret := make([]float32, 0, numRows) + for i := 0; i < numRows; i++ { + ret = append(ret, rand.Float32()) + } + return ret +} + +func generateFloat64Array(numRows int) []float64 { + ret := make([]float64, 0, numRows) + for i := 0; i < numRows; i++ { + ret = append(ret, rand.Float64()) + } + return ret +} + +func generateStringArray(numRows int) []string { + ret := make([]string, 0, numRows) + for i := 0; i < numRows; i++ { + ret = append(ret, funcutil.GenRandomStr()) + } + return ret +} + +func generateFloatVectors(numRows, dim int) []float32 { + total := numRows * dim + ret := make([]float32, 0, total) + for i := 0; i < total; i++ { + ret = append(ret, rand.Float32()) + } + return ret +} + +func generateBinaryVectors(numRows, dim int) []byte { + total := (numRows * dim) / 8 + ret := make([]byte, total) + _, err := rand.Read(ret) + if err != nil { + panic(err) + } + return ret +} + +func genFieldData(dtype schemapb.DataType, numRows, dim int) storage.FieldData { + switch dtype { + case schemapb.DataType_Bool: + return &storage.BoolFieldData{ + Data: generateBoolArray(numRows), + } + case schemapb.DataType_Int8: + return &storage.Int8FieldData{ + Data: generateInt8Array(numRows), + } + case schemapb.DataType_Int16: + return &storage.Int16FieldData{ + Data: generateInt16Array(numRows), + } + case schemapb.DataType_Int32: + return &storage.Int32FieldData{ + Data: generateInt32Array(numRows), + } + case schemapb.DataType_Int64: + return &storage.Int64FieldData{ + Data: generateInt64Array(numRows), + } + case schemapb.DataType_Float: + return &storage.FloatFieldData{ + Data: generateFloat32Array(numRows), + } + case schemapb.DataType_Double: + return &storage.DoubleFieldData{ + Data: generateFloat64Array(numRows), + } + case schemapb.DataType_String: + return &storage.StringFieldData{ + Data: generateStringArray(numRows), + } + case schemapb.DataType_VarChar: + return &storage.StringFieldData{ + Data: generateStringArray(numRows), + } + case schemapb.DataType_BinaryVector: + return &storage.BinaryVectorFieldData{ + Dim: dim, + Data: generateBinaryVectors(numRows, dim), + } + case schemapb.DataType_FloatVector: + return &storage.FloatVectorFieldData{ + Data: generateFloatVectors(numRows, dim), + Dim: dim, + } + default: + return nil + } +} + +func genScalarIndexCases(dtype schemapb.DataType) []indexTestCase { + return []indexTestCase{ + { + dtype: dtype, + typeParams: nil, + indexParams: map[string]string{ + "index_type": "inverted_index", + }, + }, + { + dtype: dtype, + typeParams: nil, + indexParams: map[string]string{ + "index_type": "flat", + }, + }, + } +} + +func genStringIndexCases(dtype schemapb.DataType) []indexTestCase { + return []indexTestCase{ + { + dtype: dtype, + typeParams: nil, + indexParams: map[string]string{ + "index_type": "inverted_index", + }, + }, + { + dtype: dtype, + typeParams: nil, + indexParams: map[string]string{ + "index_type": "marisa-trie", + }, + }, + } +} + +func genFloatVecIndexCases(dtype schemapb.DataType) []indexTestCase { + return []indexTestCase{ + { + dtype: dtype, + typeParams: nil, + indexParams: map[string]string{ + "index_type": IndexFaissIVFPQ, + "metric_type": L2, + "dim": strconv.Itoa(dim), + "nlist": strconv.Itoa(nlist), + "m": strconv.Itoa(m), + "nbits": strconv.Itoa(nbits), + "SLICE_SIZE": strconv.Itoa(sliceSize), + }, + }, + { + dtype: dtype, + typeParams: nil, + indexParams: map[string]string{ + "index_type": IndexFaissIVFFlat, + "metric_type": L2, + "dim": strconv.Itoa(dim), + "nlist": strconv.Itoa(nlist), + "SLICE_SIZE": strconv.Itoa(sliceSize), + }, + }, + } +} + +func genBinaryVecIndexCases(dtype schemapb.DataType) []indexTestCase { + return []indexTestCase{ + { + dtype: dtype, + typeParams: nil, + indexParams: map[string]string{ + "index_type": IndexFaissBinIVFFlat, + "metric_type": Jaccard, + "dim": strconv.Itoa(dim), + "nlist": strconv.Itoa(nlist), + "nbits": strconv.Itoa(nbits), + "SLICE_SIZE": strconv.Itoa(sliceSize), + }, + }, + } +} + +func genTypedIndexCase(dtype schemapb.DataType) []indexTestCase { + switch dtype { + case schemapb.DataType_Bool: + return genScalarIndexCases(dtype) + case schemapb.DataType_Int8: + return genScalarIndexCases(dtype) + case schemapb.DataType_Int16: + return genScalarIndexCases(dtype) + case schemapb.DataType_Int32: + return genScalarIndexCases(dtype) + case schemapb.DataType_Int64: + return genScalarIndexCases(dtype) + case schemapb.DataType_Float: + return genScalarIndexCases(dtype) + case schemapb.DataType_Double: + return genScalarIndexCases(dtype) + case schemapb.DataType_String: + return genScalarIndexCases(dtype) + case schemapb.DataType_VarChar: + return genStringIndexCases(dtype) + case schemapb.DataType_BinaryVector: + return genBinaryVecIndexCases(dtype) + case schemapb.DataType_FloatVector: + return genFloatVecIndexCases(dtype) + default: + return nil + } +} + +func genIndexCase() []indexTestCase { + dtypes := []schemapb.DataType{ + schemapb.DataType_Bool, + schemapb.DataType_Int8, + schemapb.DataType_Int16, + schemapb.DataType_Int32, + schemapb.DataType_Int64, + schemapb.DataType_Float, + schemapb.DataType_Double, + schemapb.DataType_String, + schemapb.DataType_VarChar, + schemapb.DataType_BinaryVector, + schemapb.DataType_FloatVector, + } + var ret []indexTestCase + for _, dtype := range dtypes { + ret = append(ret, genTypedIndexCase(dtype)...) + } + return ret +} + +func TestCgoIndex(t *testing.T) { + for _, testCase := range genIndexCase() { + index, err := NewCgoIndex(testCase.dtype, testCase.typeParams, testCase.indexParams) + assert.NoError(t, err, testCase) + + dataset := GenDataset(genFieldData(testCase.dtype, nb, dim)) + assert.NoError(t, index.Build(dataset), testCase) + + blobs, err := index.Serialize() + assert.NoError(t, err, testCase) + + copyIndex, err := NewCgoIndex(testCase.dtype, testCase.typeParams, testCase.indexParams) + assert.NoError(t, err, testCase) + + assert.NoError(t, copyIndex.Load(blobs), testCase) + } +} diff --git a/internal/util/indexcgowrapper/dataset.go b/internal/util/indexcgowrapper/dataset.go new file mode 100644 index 0000000000000000000000000000000000000000..8fd237739bac2b3d5460bed9dc613c579b003d2a --- /dev/null +++ b/internal/util/indexcgowrapper/dataset.go @@ -0,0 +1,103 @@ +package indexcgowrapper + +import ( + "github.com/milvus-io/milvus/internal/proto/schemapb" + "github.com/milvus-io/milvus/internal/storage" +) + +const ( + keyRawArr = "key_raw_arr" +) + +type Dataset struct { + DType schemapb.DataType + Data map[string]interface{} +} + +func GenFloatVecDataset(vectors []float32) *Dataset { + return &Dataset{ + DType: schemapb.DataType_FloatVector, + Data: map[string]interface{}{ + keyRawArr: vectors, + }, + } +} + +func GenBinaryVecDataset(vectors []byte) *Dataset { + return &Dataset{ + DType: schemapb.DataType_BinaryVector, + Data: map[string]interface{}{ + keyRawArr: vectors, + }, + } +} + +func GenDataset(data storage.FieldData) *Dataset { + switch f := data.(type) { + case *storage.BoolFieldData: + return &Dataset{ + DType: schemapb.DataType_Bool, + Data: map[string]interface{}{ + keyRawArr: f.Data, + }, + } + case *storage.Int8FieldData: + return &Dataset{ + DType: schemapb.DataType_Int8, + Data: map[string]interface{}{ + keyRawArr: f.Data, + }, + } + case *storage.Int16FieldData: + return &Dataset{ + DType: schemapb.DataType_Int16, + Data: map[string]interface{}{ + keyRawArr: f.Data, + }, + } + case *storage.Int32FieldData: + return &Dataset{ + DType: schemapb.DataType_Int32, + Data: map[string]interface{}{ + keyRawArr: f.Data, + }, + } + case *storage.Int64FieldData: + return &Dataset{ + DType: schemapb.DataType_Int64, + Data: map[string]interface{}{ + keyRawArr: f.Data, + }, + } + case *storage.FloatFieldData: + return &Dataset{ + DType: schemapb.DataType_Float, + Data: map[string]interface{}{ + keyRawArr: f.Data, + }, + } + case *storage.DoubleFieldData: + return &Dataset{ + DType: schemapb.DataType_Double, + Data: map[string]interface{}{ + keyRawArr: f.Data, + }, + } + case *storage.StringFieldData: + return &Dataset{ + DType: schemapb.DataType_String, + Data: map[string]interface{}{ + keyRawArr: f.Data, + }, + } + case *storage.BinaryVectorFieldData: + return GenBinaryVecDataset(f.Data) + case *storage.FloatVectorFieldData: + return GenFloatVecDataset(f.Data) + default: + return &Dataset{ + DType: schemapb.DataType_None, + Data: nil, + } + } +} diff --git a/internal/util/indexcgowrapper/helper.go b/internal/util/indexcgowrapper/helper.go new file mode 100644 index 0000000000000000000000000000000000000000..d1a4559b35d5986f5f1a2a66cf41bbd9b3c8ee1b --- /dev/null +++ b/internal/util/indexcgowrapper/helper.go @@ -0,0 +1,76 @@ +package indexcgowrapper + +/* + +#cgo CFLAGS: -I${SRCDIR}/../../core/output/include + +#cgo darwin LDFLAGS: -L${SRCDIR}/../../core/output/lib -lmilvus_indexbuilder -lmilvus_common -Wl,-rpath,"${SRCDIR}/../../core/output/lib" +#cgo linux LDFLAGS: -L${SRCDIR}/../../core/output/lib -lmilvus_indexbuilder -lmilvus_common -Wl,-rpath=${SRCDIR}/../../core/output/lib +#cgo windows LDFLAGS: -L${SRCDIR}/../../core/output/lib -lmilvus_indexbuilder -lmilvus_common -Wl,-rpath=${SRCDIR}/../../core/output/lib + +#include // free +#include "indexbuilder/index_c.h" + +*/ +import "C" +import ( + "errors" + "fmt" + "unsafe" + + "github.com/milvus-io/milvus/internal/log" + "github.com/milvus-io/milvus/internal/proto/commonpb" +) + +func GetBinarySetKeys(cBinarySet C.CBinarySet) ([]string, error) { + size := int(C.GetBinarySetSize(cBinarySet)) + if size == 0 { + return nil, fmt.Errorf("BinarySet size is zero!") + } + datas := make([]unsafe.Pointer, size) + + C.GetBinarySetKeys(cBinarySet, unsafe.Pointer(&datas[0])) + ret := make([]string, size) + for i := 0; i < size; i++ { + ret[i] = C.GoString((*C.char)(datas[i])) + } + + return ret, nil +} + +func GetBinarySetValue(cBinarySet C.CBinarySet, key string) ([]byte, error) { + cIndexKey := C.CString(key) + defer C.free(unsafe.Pointer(cIndexKey)) + ret := C.GetBinarySetValueSize(cBinarySet, cIndexKey) + size := int(ret) + if size == 0 { + return nil, fmt.Errorf("GetBinarySetValueSize size is zero!") + } + value := make([]byte, size) + status := C.CopyBinarySetValue(unsafe.Pointer(&value[0]), cIndexKey, cBinarySet) + + if err := HandleCStatus(&status, "CopyBinarySetValue failed"); err != nil { + return nil, err + } + + return value, nil +} + +// HandleCStatus deal with the error returned from CGO +func HandleCStatus(status *C.CStatus, extraInfo string) error { + if status.error_code == 0 { + return nil + } + errorCode := status.error_code + errorName, ok := commonpb.ErrorCode_name[int32(errorCode)] + if !ok { + errorName = "UnknownError" + } + errorMsg := C.GoString(status.error_msg) + defer C.free(unsafe.Pointer(status.error_msg)) + + finalMsg := fmt.Sprintf("[%s] %s", errorName, errorMsg) + logMsg := fmt.Sprintf("%s, C Runtime Exception: %s\n", extraInfo, finalMsg) + log.Warn(logMsg) + return errors.New(finalMsg) +} diff --git a/internal/util/indexcgowrapper/index.go b/internal/util/indexcgowrapper/index.go new file mode 100644 index 0000000000000000000000000000000000000000..2789ded52f06c18ab03c89159bfad55f46e117eb --- /dev/null +++ b/internal/util/indexcgowrapper/index.go @@ -0,0 +1,267 @@ +package indexcgowrapper + +/* + +#cgo CFLAGS: -I${SRCDIR}/../../core/output/include + +#cgo darwin LDFLAGS: -L${SRCDIR}/../../core/output/lib -lmilvus_indexbuilder -lmilvus_common -Wl,-rpath,"${SRCDIR}/../../core/output/lib" +#cgo linux LDFLAGS: -L${SRCDIR}/../../core/output/lib -lmilvus_indexbuilder -lmilvus_common -Wl,-rpath=${SRCDIR}/../../core/output/lib +#cgo windows LDFLAGS: -L${SRCDIR}/../core/output/lib -lmilvus_common -lmilvus_indexbuilder -Wl,-rpath=${SRCDIR}/../core/output/lib + +#include // free +#include "indexbuilder/index_c.h" + +*/ +import "C" +import ( + "fmt" + "path/filepath" + "runtime" + "unsafe" + + "github.com/golang/protobuf/proto" + + "github.com/milvus-io/milvus/internal/proto/commonpb" + "github.com/milvus-io/milvus/internal/proto/schemapb" + + "github.com/milvus-io/milvus/internal/log" + "github.com/milvus-io/milvus/internal/proto/indexcgopb" + "github.com/milvus-io/milvus/internal/storage" +) + +type Blob = storage.Blob + +type CodecIndex interface { + Build(*Dataset) error + Serialize() ([]*Blob, error) + Load([]*Blob) error + Delete() error +} + +var ( + _ CodecIndex = (*CgoIndex)(nil) +) + +type CgoIndex struct { + indexPtr C.CIndex + close bool +} + +// TODO: use proto.Marshal instead of proto.MarshalTextString for better compatibility. +func NewCgoIndex(dtype schemapb.DataType, typeParams, indexParams map[string]string) (*CgoIndex, error) { + protoTypeParams := &indexcgopb.TypeParams{ + Params: make([]*commonpb.KeyValuePair, 0), + } + for key, value := range typeParams { + protoTypeParams.Params = append(protoTypeParams.Params, &commonpb.KeyValuePair{Key: key, Value: value}) + } + typeParamsStr := proto.MarshalTextString(protoTypeParams) + + protoIndexParams := &indexcgopb.IndexParams{ + Params: make([]*commonpb.KeyValuePair, 0), + } + for key, value := range indexParams { + protoIndexParams.Params = append(protoIndexParams.Params, &commonpb.KeyValuePair{Key: key, Value: value}) + } + indexParamsStr := proto.MarshalTextString(protoIndexParams) + + typeParamsPointer := C.CString(typeParamsStr) + indexParamsPointer := C.CString(indexParamsStr) + defer C.free(unsafe.Pointer(typeParamsPointer)) + defer C.free(unsafe.Pointer(indexParamsPointer)) + + var indexPtr C.CIndex + cintDType := uint32(dtype) + status := C.CreateIndex(cintDType, typeParamsPointer, indexParamsPointer, &indexPtr) + if err := HandleCStatus(&status, "failed to create index"); err != nil { + return nil, err + } + + index := &CgoIndex{ + indexPtr: indexPtr, + close: false, + } + + runtime.SetFinalizer(index, func(index *CgoIndex) { + if index != nil && !index.close { + log.Error("there is leakage in index object, please check.") + } + }) + + return index, nil +} + +func (index *CgoIndex) Build(dataset *Dataset) error { + switch dataset.DType { + case schemapb.DataType_None: + return fmt.Errorf("build index on supported data type: %s", dataset.DType.String()) + case schemapb.DataType_FloatVector: + return index.buildFloatVecIndex(dataset) + case schemapb.DataType_BinaryVector: + return index.buildBinaryVecIndex(dataset) + case schemapb.DataType_Bool: + return index.buildBoolIndex(dataset) + case schemapb.DataType_Int8: + return index.buildInt8Index(dataset) + case schemapb.DataType_Int16: + return index.buildInt16Index(dataset) + case schemapb.DataType_Int32: + return index.buildInt32Index(dataset) + case schemapb.DataType_Int64: + return index.buildInt64Index(dataset) + case schemapb.DataType_Float: + return index.buildFloatIndex(dataset) + case schemapb.DataType_Double: + return index.buildDoubleIndex(dataset) + case schemapb.DataType_String: + return index.buildStringIndex(dataset) + case schemapb.DataType_VarChar: + return index.buildStringIndex(dataset) + default: + return fmt.Errorf("build index on unsupported data type: %s", dataset.DType.String()) + } +} + +func (index *CgoIndex) buildFloatVecIndex(dataset *Dataset) error { + vectors := dataset.Data[keyRawArr].([]float32) + status := C.BuildFloatVecIndex(index.indexPtr, (C.int64_t)(len(vectors)), (*C.float)(&vectors[0])) + return HandleCStatus(&status, "failed to build float vector index") +} + +func (index *CgoIndex) buildBinaryVecIndex(dataset *Dataset) error { + vectors := dataset.Data[keyRawArr].([]byte) + status := C.BuildBinaryVecIndex(index.indexPtr, (C.int64_t)(len(vectors)), (*C.uint8_t)(&vectors[0])) + return HandleCStatus(&status, "failed to build binary vector index") +} + +// TODO: investigate if we can pass an bool array to cgo. +func (index *CgoIndex) buildBoolIndex(dataset *Dataset) error { + arr := dataset.Data[keyRawArr].([]bool) + f := &schemapb.BoolArray{ + Data: arr, + } + data, err := proto.Marshal(f) + if err != nil { + return err + } + status := C.BuildScalarIndex(index.indexPtr, (C.int64_t)(len(data)), unsafe.Pointer(&data[0])) + return HandleCStatus(&status, "failed to build scalar index") +} + +// TODO: refactor these duplicated code after generic programming is supported. + +func (index *CgoIndex) buildInt8Index(dataset *Dataset) error { + data := dataset.Data[keyRawArr].([]int8) + status := C.BuildScalarIndex(index.indexPtr, (C.int64_t)(len(data)), unsafe.Pointer(&data[0])) + return HandleCStatus(&status, "failed to build scalar index") +} + +func (index *CgoIndex) buildInt16Index(dataset *Dataset) error { + data := dataset.Data[keyRawArr].([]int16) + status := C.BuildScalarIndex(index.indexPtr, (C.int64_t)(len(data)), unsafe.Pointer(&data[0])) + return HandleCStatus(&status, "failed to build scalar index") +} + +func (index *CgoIndex) buildInt32Index(dataset *Dataset) error { + data := dataset.Data[keyRawArr].([]int32) + status := C.BuildScalarIndex(index.indexPtr, (C.int64_t)(len(data)), unsafe.Pointer(&data[0])) + return HandleCStatus(&status, "failed to build scalar index") +} + +func (index *CgoIndex) buildInt64Index(dataset *Dataset) error { + data := dataset.Data[keyRawArr].([]int64) + status := C.BuildScalarIndex(index.indexPtr, (C.int64_t)(len(data)), unsafe.Pointer(&data[0])) + return HandleCStatus(&status, "failed to build scalar index") +} + +func (index *CgoIndex) buildFloatIndex(dataset *Dataset) error { + data := dataset.Data[keyRawArr].([]float32) + status := C.BuildScalarIndex(index.indexPtr, (C.int64_t)(len(data)), unsafe.Pointer(&data[0])) + return HandleCStatus(&status, "failed to build scalar index") +} + +func (index *CgoIndex) buildDoubleIndex(dataset *Dataset) error { + data := dataset.Data[keyRawArr].([]float64) + status := C.BuildScalarIndex(index.indexPtr, (C.int64_t)(len(data)), unsafe.Pointer(&data[0])) + return HandleCStatus(&status, "failed to build scalar index") +} + +func (index *CgoIndex) buildStringIndex(dataset *Dataset) error { + arr := dataset.Data[keyRawArr].([]string) + f := &schemapb.StringArray{ + Data: arr, + } + data, err := proto.Marshal(f) + if err != nil { + return err + } + status := C.BuildScalarIndex(index.indexPtr, (C.int64_t)(len(data)), unsafe.Pointer(&data[0])) + return HandleCStatus(&status, "failed to build scalar index") +} + +func (index *CgoIndex) Serialize() ([]*Blob, error) { + var cBinarySet C.CBinarySet + + status := C.SerializeIndexToBinarySet(index.indexPtr, &cBinarySet) + defer func() { + if cBinarySet != nil { + C.DeleteBinarySet(cBinarySet) + } + }() + if err := HandleCStatus(&status, "failed to serialize index to binary set"); err != nil { + return nil, err + } + + keys, err := GetBinarySetKeys(cBinarySet) + if err != nil { + return nil, err + } + ret := make([]*Blob, 0) + for _, key := range keys { + value, err := GetBinarySetValue(cBinarySet, key) + if err != nil { + return nil, err + } + blob := &Blob{ + Key: key, + Value: value, + } + ret = append(ret, blob) + } + + return ret, nil +} + +func (index *CgoIndex) Load(blobs []*Blob) error { + var cBinarySet C.CBinarySet + status := C.NewBinarySet(&cBinarySet) + defer C.DeleteBinarySet(cBinarySet) + + if err := HandleCStatus(&status, "failed to load index"); err != nil { + return err + } + for _, blob := range blobs { + key := blob.Key + byteIndex := blob.Value + indexPtr := unsafe.Pointer(&byteIndex[0]) + indexLen := C.int64_t(len(byteIndex)) + binarySetKey := filepath.Base(key) + indexKey := C.CString(binarySetKey) + status = C.AppendIndexBinary(cBinarySet, indexPtr, indexLen, indexKey) + C.free(unsafe.Pointer(indexKey)) + if err := HandleCStatus(&status, "failed to load index"); err != nil { + return err + } + } + status = C.LoadIndexFromBinarySet(index.indexPtr, cBinarySet) + return HandleCStatus(&status, "failed to load index") +} + +func (index *CgoIndex) Delete() error { + if index.close { + return nil + } + status := C.DeleteIndex(index.indexPtr) + index.close = true + return HandleCStatus(&status, "failed to delete index") +} diff --git a/internal/util/indexcgowrapper/index_bench.go b/internal/util/indexcgowrapper/index_bench.go new file mode 100644 index 0000000000000000000000000000000000000000..b550f805790eb3030d339ec18c541829f32d019c --- /dev/null +++ b/internal/util/indexcgowrapper/index_bench.go @@ -0,0 +1,6 @@ +//go:build linux +// +build linux + +package indexcgowrapper + +// TODO: add a benchmark to check if any leakage in cgo. diff --git a/internal/util/indexcgowrapper/index_test.go b/internal/util/indexcgowrapper/index_test.go new file mode 100644 index 0000000000000000000000000000000000000000..f29184c0bcb5a966c0a8fb48f381f2e7973fb31c --- /dev/null +++ b/internal/util/indexcgowrapper/index_test.go @@ -0,0 +1,329 @@ +//go:build linux +// +build linux + +package indexcgowrapper + +import ( + "fmt" + "strconv" + "testing" + + "github.com/milvus-io/milvus/internal/proto/schemapb" + + "github.com/stretchr/testify/assert" +) + +const ( + // index type + IndexFaissIDMap = "FLAT" + IndexFaissIVFFlat = "IVF_FLAT" + IndexFaissIVFPQ = "IVF_PQ" + IndexFaissIVFSQ8 = "IVF_SQ8" + IndexFaissIVFSQ8H = "IVF_SQ8_HYBRID" + IndexFaissBinIDMap = "BIN_FLAT" + IndexFaissBinIVFFlat = "BIN_IVF_FLAT" + IndexNsg = "NSG" + + IndexHNSW = "HNSW" + IndexRHNSWFlat = "RHNSW_FLAT" + IndexRHNSWPQ = "RHNSW_PQ" + IndexRHNSWSQ = "RHNSW_SQ" + IndexANNOY = "ANNOY" + IndexNGTPANNG = "NGT_PANNG" + IndexNGTONNG = "NGT_ONNG" + + // metric type + L2 = "L2" + IP = "IP" + hamming = "HAMMING" + Jaccard = "JACCARD" + tanimoto = "TANIMOTO" + + dim = 8 + nlist = 100 + m = 4 + nbits = 8 + nb = 1000 + nprobe = 8 + sliceSize = 4 + efConstruction = 200 + ef = 200 + edgeSize = 10 + epsilon = 0.1 + maxSearchEdges = 50 +) + +type vecTestCase struct { + indexType string + metricType string + isBinary bool + dtype schemapb.DataType +} + +func generateFloatVectorTestCases() []vecTestCase { + return []vecTestCase{ + {IndexFaissIDMap, L2, false, schemapb.DataType_FloatVector}, + {IndexFaissIDMap, IP, false, schemapb.DataType_FloatVector}, + {IndexFaissIVFFlat, L2, false, schemapb.DataType_FloatVector}, + {IndexFaissIVFFlat, IP, false, schemapb.DataType_FloatVector}, + {IndexFaissIVFPQ, L2, false, schemapb.DataType_FloatVector}, + {IndexFaissIVFPQ, IP, false, schemapb.DataType_FloatVector}, + {IndexFaissIVFSQ8, L2, false, schemapb.DataType_FloatVector}, + {IndexFaissIVFSQ8, IP, false, schemapb.DataType_FloatVector}, + //{IndexFaissIVFSQ8H, L2, false, schemapb.DataType_FloatVector}, // TODO: enable gpu + //{IndexFaissIVFSQ8H, IP, false, schemapb.DataType_FloatVector}, + {IndexNsg, L2, false, schemapb.DataType_FloatVector}, + {IndexNsg, IP, false, schemapb.DataType_FloatVector}, + {IndexHNSW, L2, false, schemapb.DataType_FloatVector}, + {IndexHNSW, IP, false, schemapb.DataType_FloatVector}, + {IndexRHNSWFlat, L2, false, schemapb.DataType_FloatVector}, + {IndexRHNSWFlat, IP, false, schemapb.DataType_FloatVector}, + {IndexRHNSWPQ, L2, false, schemapb.DataType_FloatVector}, + {IndexRHNSWPQ, IP, false, schemapb.DataType_FloatVector}, + {IndexRHNSWSQ, L2, false, schemapb.DataType_FloatVector}, + {IndexRHNSWSQ, IP, false, schemapb.DataType_FloatVector}, + {IndexANNOY, L2, false, schemapb.DataType_FloatVector}, + {IndexANNOY, IP, false, schemapb.DataType_FloatVector}, + {IndexNGTPANNG, L2, false, schemapb.DataType_FloatVector}, + {IndexNGTPANNG, IP, false, schemapb.DataType_FloatVector}, + {IndexNGTONNG, L2, false, schemapb.DataType_FloatVector}, + {IndexNGTONNG, IP, false, schemapb.DataType_FloatVector}, + } +} + +func generateBinaryVectorTestCases() []vecTestCase { + return []vecTestCase{ + {IndexFaissBinIVFFlat, Jaccard, true, schemapb.DataType_BinaryVector}, + {IndexFaissBinIVFFlat, hamming, true, schemapb.DataType_BinaryVector}, + {IndexFaissBinIVFFlat, tanimoto, true, schemapb.DataType_BinaryVector}, + {IndexFaissBinIDMap, Jaccard, true, schemapb.DataType_BinaryVector}, + {IndexFaissBinIDMap, hamming, true, schemapb.DataType_BinaryVector}, + } +} + +func generateTestCases() []vecTestCase { + return append(generateFloatVectorTestCases(), generateBinaryVectorTestCases()...) +} + +func generateParams(indexType, metricType string) (map[string]string, map[string]string) { + typeParams := make(map[string]string) + indexParams := make(map[string]string) + indexParams["index_type"] = indexType + indexParams["metric_type"] = metricType + if indexType == IndexFaissIDMap { // float vector + indexParams["dim"] = strconv.Itoa(dim) + indexParams["SLICE_SIZE"] = strconv.Itoa(sliceSize) + } else if indexType == IndexFaissIVFFlat { + indexParams["dim"] = strconv.Itoa(dim) + indexParams["nlist"] = strconv.Itoa(nlist) + } else if indexType == IndexFaissIVFPQ { + indexParams["dim"] = strconv.Itoa(dim) + indexParams["nlist"] = strconv.Itoa(nlist) + indexParams["m"] = strconv.Itoa(m) + indexParams["nbits"] = strconv.Itoa(nbits) + indexParams["SLICE_SIZE"] = strconv.Itoa(sliceSize) + } else if indexType == IndexFaissIVFSQ8 { + indexParams["dim"] = strconv.Itoa(dim) + indexParams["nlist"] = strconv.Itoa(nlist) + indexParams["nbits"] = strconv.Itoa(nbits) + indexParams["SLICE_SIZE"] = strconv.Itoa(sliceSize) + } else if indexType == IndexFaissIVFSQ8H { + // TODO: enable gpu + } else if indexType == IndexNsg { + indexParams["dim"] = strconv.Itoa(dim) + indexParams["nlist"] = strconv.Itoa(163) + indexParams["nprobe"] = strconv.Itoa(nprobe) + indexParams["knng"] = strconv.Itoa(20) + indexParams["search_length"] = strconv.Itoa(40) + indexParams["out_degree"] = strconv.Itoa(30) + indexParams["candidate_pool_size"] = strconv.Itoa(100) + } else if indexType == IndexHNSW { + indexParams["dim"] = strconv.Itoa(dim) + indexParams["M"] = strconv.Itoa(16) + indexParams["efConstruction"] = strconv.Itoa(efConstruction) + indexParams["ef"] = strconv.Itoa(ef) + } else if indexType == IndexRHNSWFlat { + indexParams["dim"] = strconv.Itoa(dim) + indexParams["M"] = strconv.Itoa(16) + indexParams["efConstruction"] = strconv.Itoa(efConstruction) + indexParams["ef"] = strconv.Itoa(ef) + indexParams["SLICE_SIZE"] = strconv.Itoa(sliceSize) + } else if indexType == IndexRHNSWPQ { + indexParams["dim"] = strconv.Itoa(dim) + indexParams["M"] = strconv.Itoa(16) + indexParams["efConstruction"] = strconv.Itoa(efConstruction) + indexParams["ef"] = strconv.Itoa(ef) + indexParams["SLICE_SIZE"] = strconv.Itoa(sliceSize) + indexParams["PQM"] = strconv.Itoa(8) + } else if indexType == IndexRHNSWSQ { + indexParams["dim"] = strconv.Itoa(dim) + indexParams["M"] = strconv.Itoa(16) + indexParams["efConstruction"] = strconv.Itoa(efConstruction) + indexParams["ef"] = strconv.Itoa(ef) + indexParams["SLICE_SIZE"] = strconv.Itoa(sliceSize) + } else if indexType == IndexANNOY { + indexParams["dim"] = strconv.Itoa(dim) + indexParams["n_trees"] = strconv.Itoa(4) + indexParams["search_k"] = strconv.Itoa(100) + indexParams["SLICE_SIZE"] = strconv.Itoa(sliceSize) + } else if indexType == IndexNGTPANNG { + indexParams["dim"] = strconv.Itoa(dim) + indexParams["edge_size"] = strconv.Itoa(edgeSize) + indexParams["epsilon"] = fmt.Sprint(epsilon) + indexParams["max_search_edges"] = strconv.Itoa(maxSearchEdges) + indexParams["forcedly_pruned_edge_size"] = strconv.Itoa(60) + indexParams["selectively_pruned_edge_size"] = strconv.Itoa(30) + indexParams["SLICE_SIZE"] = strconv.Itoa(sliceSize) + } else if indexType == IndexNGTONNG { + indexParams["dim"] = strconv.Itoa(dim) + indexParams["edge_size"] = strconv.Itoa(edgeSize) + indexParams["epsilon"] = fmt.Sprint(epsilon) + indexParams["max_search_edges"] = strconv.Itoa(maxSearchEdges) + indexParams["outgoing_edge_size"] = strconv.Itoa(5) + indexParams["incoming_edge_size"] = strconv.Itoa(40) + indexParams["SLICE_SIZE"] = strconv.Itoa(sliceSize) + } else if indexType == IndexFaissBinIVFFlat { // binary vector + indexParams["dim"] = strconv.Itoa(dim) + indexParams["nlist"] = strconv.Itoa(nlist) + indexParams["m"] = strconv.Itoa(m) + indexParams["nbits"] = strconv.Itoa(nbits) + indexParams["SLICE_SIZE"] = strconv.Itoa(sliceSize) + } else if indexType == IndexFaissBinIDMap { + indexParams["dim"] = strconv.Itoa(dim) + } else { + panic("") + } + + return typeParams, indexParams +} + +func TestCIndex_New(t *testing.T) { + for _, c := range generateTestCases() { + typeParams, indexParams := generateParams(c.indexType, c.metricType) + + index, err := NewCgoIndex(c.dtype, typeParams, indexParams) + assert.Equal(t, err, nil) + assert.NotEqual(t, index, nil) + + err = index.Delete() + assert.Equal(t, err, nil) + } +} + +func TestCIndex_BuildFloatVecIndex(t *testing.T) { + for _, c := range generateFloatVectorTestCases() { + typeParams, indexParams := generateParams(c.indexType, c.metricType) + + index, err := NewCgoIndex(c.dtype, typeParams, indexParams) + assert.Equal(t, err, nil) + assert.NotEqual(t, index, nil) + + vectors := generateFloatVectors(nb, dim) + err = index.Build(GenFloatVecDataset(vectors)) + assert.Equal(t, err, nil) + + err = index.Delete() + assert.Equal(t, err, nil) + } +} + +func TestCIndex_BuildBinaryVecIndex(t *testing.T) { + for _, c := range generateBinaryVectorTestCases() { + typeParams, indexParams := generateParams(c.indexType, c.metricType) + + index, err := NewCgoIndex(c.dtype, typeParams, indexParams) + assert.Equal(t, err, nil) + assert.NotEqual(t, index, nil) + + vectors := generateBinaryVectors(nb, dim) + err = index.Build(GenBinaryVecDataset(vectors)) + assert.Equal(t, err, nil) + + err = index.Delete() + assert.Equal(t, err, nil) + } +} + +func TestCIndex_Codec(t *testing.T) { + for _, c := range generateTestCases() { + typeParams, indexParams := generateParams(c.indexType, c.metricType) + + index, err := NewCgoIndex(c.dtype, typeParams, indexParams) + assert.Equal(t, err, nil) + assert.NotEqual(t, index, nil) + + if !c.isBinary { + vectors := generateFloatVectors(nb, dim) + err = index.Build(GenFloatVecDataset(vectors)) + assert.Equal(t, err, nil) + } else { + vectors := generateBinaryVectors(nb, dim) + err = index.Build(GenBinaryVecDataset(vectors)) + assert.Equal(t, err, nil) + } + + blobs, err := index.Serialize() + assert.Equal(t, err, nil) + + copyIndex, err := NewCgoIndex(c.dtype, typeParams, indexParams) + assert.NotEqual(t, copyIndex, nil) + assert.Equal(t, err, nil) + err = copyIndex.Load(blobs) + assert.Equal(t, err, nil) + copyBlobs, err := copyIndex.Serialize() + assert.Equal(t, err, nil) + assert.Equal(t, len(blobs), len(copyBlobs)) + // TODO: check key, value and more + + err = index.Delete() + assert.Equal(t, err, nil) + err = copyIndex.Delete() + assert.Equal(t, err, nil) + } +} + +func TestCIndex_Delete(t *testing.T) { + for _, c := range generateTestCases() { + typeParams, indexParams := generateParams(c.indexType, c.metricType) + + index, err := NewCgoIndex(c.dtype, typeParams, indexParams) + assert.Equal(t, err, nil) + assert.NotEqual(t, index, nil) + + err = index.Delete() + assert.Equal(t, err, nil) + } +} + +func TestCIndex_Error(t *testing.T) { + indexPtr, err := NewCgoIndex(schemapb.DataType_FloatVector, nil, nil) + assert.Nil(t, err) + + t.Run("Serialize error", func(t *testing.T) { + blobs, err := indexPtr.Serialize() + assert.NotNil(t, err) + assert.Nil(t, blobs) + }) + + t.Run("Load error", func(t *testing.T) { + blobs := []*Blob{{ + Key: "test", + Value: []byte("value"), + }, + } + err = indexPtr.Load(blobs) + assert.NotNil(t, err) + }) + + t.Run("BuildFloatVecIndexWithoutIds error", func(t *testing.T) { + floatVectors := []float32{1.1, 2.2, 3.3} + err = indexPtr.Build(GenFloatVecDataset(floatVectors)) + assert.NotNil(t, err) + }) + + t.Run("BuildBinaryVecIndexWithoutIds error", func(t *testing.T) { + binaryVectors := []byte("binaryVectors") + err = indexPtr.Build(GenBinaryVecDataset(binaryVectors)) + assert.NotNil(t, err) + }) +} diff --git a/internal/util/indexparamcheck/scalar_index_checker.go b/internal/util/indexparamcheck/scalar_index_checker.go new file mode 100644 index 0000000000000000000000000000000000000000..7c23de95ed0ccdf00a6d629e087de2e0155ee8b7 --- /dev/null +++ b/internal/util/indexparamcheck/scalar_index_checker.go @@ -0,0 +1,8 @@ +package indexparamcheck + +import "github.com/milvus-io/milvus/internal/proto/schemapb" + +// TODO: check index parameters according to the index type & data type. +func CheckIndexValid(dType schemapb.DataType, indexType IndexType, indexParams map[string]string) error { + return nil +} diff --git a/internal/util/indexparamcheck/scalar_index_checker_test.go b/internal/util/indexparamcheck/scalar_index_checker_test.go new file mode 100644 index 0000000000000000000000000000000000000000..dfd6080d7e9eb8576ffe4919d5cbc271996b4019 --- /dev/null +++ b/internal/util/indexparamcheck/scalar_index_checker_test.go @@ -0,0 +1,12 @@ +package indexparamcheck + +import ( + "testing" + + "github.com/milvus-io/milvus/internal/proto/schemapb" + "github.com/stretchr/testify/assert" +) + +func TestCheckIndexValid(t *testing.T) { + assert.NoError(t, CheckIndexValid(schemapb.DataType_Int64, "inverted_index", nil)) +}