Skip to content

milvus
milvus

qq_34331598 / milvus
与 Fork 源项目一致

Fork自 milvus / milvus

通知 2
Star 0
Fork 0
milvus
milvus

体验新版 GitCode，发现更多精彩内容 >>

未验证 提交 c94699fb 编写于 作者: G groot 提交者： GitHub
浏览文件
操作

refine code (#3002) 

Signed-off-by: Nyhmo <yihua.mo@zilliz.com>
上级 26536fac
展开全部 隐藏空白更改
内联 并排
Showing with 3317 addition and 3877 deletion
core/src/db/DBImpl.cpp
浏览文件 @ c94699fb
......@@ -1780,7 +1780,6 @@ DBImpl::CreateIndex(const std::shared_ptr<server::Context>& context, const std::
// step 2: check index difference
CollectionIndex old_index;
old_index.field_name_ = index.field_name_;
status = DescribeIndex(collection_id, old_index);
if (!status.ok()) {
LOG_ENGINE_ERROR_ << "Failed to get collection index info for collection: " << collection_id;
......@@ -1790,7 +1789,8 @@ DBImpl::CreateIndex(const std::shared_ptr<server::Context>& context, const std::
// step 3: update index info
CollectionIndex new_index = index;
json new_index_json = new_index.extra_params_;
new_index.metric_type_ = old_index.metric_type_; // dont change metric type, it was defined by CreateCollection
// new_index.metric_type_ = old_index.metric_type_;
// dont change metric type, it was defined by CreateCollection
if (!utils::IsSameIndex(old_index, new_index)) {
status = UpdateCollectionIndexRecursively(collection_id, new_index);
if (!status.ok()) {
......@@ -2926,18 +2926,19 @@ DBImpl::WaitCollectionIndexRecursively(const std::shared_ptr<server::Context>& c
// for IDMAP type, only wait all NEW file converted to RAW file
// for other type, wait NEW/RAW/NEW_MERGE/NEW_INDEX/TO_INDEX files converted to INDEX files
std::vector<int> file_types;
if (utils::IsRawIndexType(index.engine_type_)) {
file_types = {
static_cast<int32_t>(meta::SegmentSchema::NEW),
static_cast<int32_t>(meta::SegmentSchema::NEW_MERGE),
};
} else {
file_types = {
static_cast<int32_t>(meta::SegmentSchema::RAW), static_cast<int32_t>(meta::SegmentSchema::NEW),
static_cast<int32_t>(meta::SegmentSchema::NEW_MERGE), static_cast<int32_t>(meta::SegmentSchema::NEW_INDEX),
static_cast<int32_t>(meta::SegmentSchema::TO_INDEX),
};
}
// if (utils::IsRawIndexType(index.engine_type_)) {
// file_types = {
// static_cast<int32_t>(meta::SegmentSchema::NEW),
// static_cast<int32_t>(meta::SegmentSchema::NEW_MERGE),
// };
// } else {
// file_types = {
// static_cast<int32_t>(meta::SegmentSchema::RAW), static_cast<int32_t>(meta::SegmentSchema::NEW),
// static_cast<int32_t>(meta::SegmentSchema::NEW_MERGE),
// static_cast<int32_t>(meta::SegmentSchema::NEW_INDEX),
// static_cast<int32_t>(meta::SegmentSchema::TO_INDEX),
// };
// }
// get files to build index
{
......@@ -2949,9 +2950,9 @@ DBImpl::WaitCollectionIndexRecursively(const std::shared_ptr<server::Context>& c
if (repeat % WAIT_BUILD_INDEX_INTERVAL == 0) {
LOG_ENGINE_DEBUG_ << files_holder.HoldFiles().size() << " non-index files detected! Will build index "
<< times;
if (!utils::IsRawIndexType(index.engine_type_)) {
status = meta_ptr_->UpdateCollectionFilesToIndex(collection_id);
}
// if (!utils::IsRawIndexType(index.engine_type_)) {
// status = meta_ptr_->UpdateCollectionFilesToIndex(collection_id);
// }
}
index_req_swn_.Wait_For(std::chrono::seconds(1));
......
core/src/db/SSDBImpl.cpp
浏览文件 @ c94699fb
......@@ -665,9 +665,6 @@ SSDBImpl::CreateIndex(const std::shared_ptr<server::Context>& context, const std
return status;
}
if (old_index.metric_type_ != (int32_t)MetricType::INVALID) {
new_index.metric_type_ = old_index.metric_type_; // dont change metric type, it was defined by CreateCollection
}
if (utils::IsSameIndex(old_index, new_index)) {
return Status::OK(); // same index
}
......
core/src/db/SnapshotUtils.cpp
浏览文件 @ c94699fb
......@@ -12,6 +12,7 @@
#include "db/SnapshotUtils.h"
#include "db/SnapshotHandlers.h"
#include "db/SnapshotVisitor.h"
#include "db/Utils.h"
#include "db/snapshot/CompoundOperations.h"
#include "db/snapshot/Resources.h"
#include "db/snapshot/Snapshots.h"
......@@ -49,11 +50,10 @@ SetSnapshotIndex(const std::string& collection_name, const std::string& field_na
snapshot::OperationContext ss_context;
if (IsVectorField(field)) {
std::string index_name = knowhere::OldIndexTypeToStr(index_info.engine_type_);
auto new_element = std::make_shared<snapshot::FieldElement>(ss->GetCollectionId(), field->GetID(), index_name,
milvus::engine::FieldElementType::FET_INDEX);
auto new_element = std::make_shared<snapshot::FieldElement>(
ss->GetCollectionId(), field->GetID(), index_info.index_name_, milvus::engine::FieldElementType::FET_INDEX);
nlohmann::json json;
json[engine::PARAM_INDEX_METRIC_TYPE] = index_info.metric_type_;
json[engine::PARAM_INDEX_METRIC_TYPE] = index_info.metric_name_;
json[engine::PARAM_INDEX_EXTRA_PARAMS] = index_info.extra_params_;
new_element->SetParams(json);
ss_context.new_field_elements.push_back(new_element);
......@@ -75,9 +75,6 @@ SetSnapshotIndex(const std::string& collection_name, const std::string& field_na
Status
GetSnapshotIndex(const std::string& collection_name, const std::string& field_name,
engine::CollectionIndex& index_info) {
index_info.engine_type_ = 0;
index_info.metric_type_ = 0;
snapshot::ScopedSnapshotT ss;
STATUS_CHECK(snapshot::Snapshots::GetInstance().GetSnapshot(ss, collection_name));
......@@ -90,11 +87,10 @@ GetSnapshotIndex(const std::string& collection_name, const std::string& field_na
if (IsVectorField(field)) {
for (auto& field_element : field_elements) {
if (field_element->GetFtype() == (int64_t)milvus::engine::FieldElementType::FET_INDEX) {
std::string index_name = field_element->GetName();
index_info.engine_type_ = knowhere::StrToOldIndexType(index_name);
index_info.index_name_ = field_element->GetName();
auto json = field_element->GetParams();
if (json.find(engine::PARAM_INDEX_METRIC_TYPE) != json.end()) {
index_info.metric_type_ = json[engine::PARAM_INDEX_METRIC_TYPE];
index_info.metric_name_ = json[engine::PARAM_INDEX_METRIC_TYPE];
}
if (json.find(engine::PARAM_INDEX_EXTRA_PARAMS) != json.end()) {
index_info.extra_params_ = json[engine::PARAM_INDEX_EXTRA_PARAMS];
......@@ -105,7 +101,7 @@ GetSnapshotIndex(const std::string& collection_name, const std::string& field_na
} else {
for (auto& field_element : field_elements) {
if (field_element->GetFtype() == (int64_t)milvus::engine::FieldElementType::FET_INDEX) {
index_info.engine_type_ = (int32_t)engine::StructuredIndexType::SORTED;
index_info.index_name_ = "SORTED";
}
}
}
......
core/src/db/Types.h
浏览文件 @ c94699fb
......@@ -41,10 +41,8 @@ typedef std::vector<faiss::Index::idx_t> ResultIds;
typedef std::vector<faiss::Index::distance_t> ResultDistances;
struct CollectionIndex {
std::string field_name_;
std::string index_name_;
int32_t engine_type_ = (int)EngineType::FAISS_IDMAP;
int32_t metric_type_ = (int)MetricType::L2;
std::string metric_name_;
milvus::json extra_params_ = {{"nlist", 2048}};
};
......
core/src/db/Utils.cpp
浏览文件 @ c94699fb
......@@ -168,8 +168,8 @@ GetParentPath(const std::string& path, std::string& parent_path) {
bool
IsSameIndex(const CollectionIndex& index1, const CollectionIndex& index2) {
return index1.engine_type_ == index2.engine_type_ && index1.extra_params_ == index2.extra_params_ &&
index1.metric_type_ == index2.metric_type_;
return index1.index_name_ == index2.index_name_ && index1.extra_params_ == index2.extra_params_ &&
index1.metric_name_ == index2.metric_name_;
}
bool
......@@ -302,6 +302,27 @@ EraseFromCache(const std::string& item_key) {
}
#endif
}
std::string
IndexTypeToStr(const int32_t type) {
auto pair = s_index_type2name.find(type);
if (pair == s_index_type2name.end()) {
return "";
}
return pair->second;
}
int32_t
StrToIndexType(const std::string& str) {
auto pair = s_index_name2type.find(str);
if (pair == s_index_name2type.end()) {
return 0;
}
return pair->second;
}
} // namespace utils
} // namespace engine
} // namespace milvus
core/src/db/Utils.h
浏览文件 @ c94699fb
......@@ -86,6 +86,12 @@ ExitOnWriteError(Status& status);
void
EraseFromCache(const std::string& item_key);
std::string
IndexTypeToStr(const int32_t type);
int32_t
StrToIndexType(const std::string& str);
} // namespace utils
} // namespace engine
} // namespace milvus
core/src/db/meta/MetaTypes.h
浏览文件 @ c94699fb
......@@ -47,23 +47,42 @@ enum class EngineType {
MAX_VALUE = HNSW_SQ8NM,
};
static std::map<std::string, EngineType> s_map_engine_type = {
{knowhere::IndexEnum::INDEX_FAISS_IDMAP, EngineType::FAISS_IDMAP},
{knowhere::IndexEnum::INDEX_FAISS_IVFFLAT, EngineType::FAISS_IVFFLAT},
{knowhere::IndexEnum::INDEX_FAISS_IVFPQ, EngineType::FAISS_PQ},
{knowhere::IndexEnum::INDEX_FAISS_IVFSQ8, EngineType::FAISS_IVFSQ8},
{knowhere::IndexEnum::INDEX_FAISS_IVFSQ8NR, EngineType::FAISS_IVFSQ8NR},
{knowhere::IndexEnum::INDEX_FAISS_IVFSQ8H, EngineType::FAISS_IVFSQ8H},
{knowhere::IndexEnum::INDEX_NSG, EngineType::NSG_MIX},
static std::map<std::string, int32_t> s_index_name2type = {
{knowhere::IndexEnum::INDEX_FAISS_IDMAP, (int32_t)EngineType::FAISS_IDMAP},
{knowhere::IndexEnum::INDEX_FAISS_IVFFLAT, (int32_t)EngineType::FAISS_IVFFLAT},
{knowhere::IndexEnum::INDEX_FAISS_IVFPQ, (int32_t)EngineType::FAISS_PQ},
{knowhere::IndexEnum::INDEX_FAISS_IVFSQ8, (int32_t)EngineType::FAISS_IVFSQ8},
{knowhere::IndexEnum::INDEX_FAISS_IVFSQ8NR, (int32_t)EngineType::FAISS_IVFSQ8NR},
{knowhere::IndexEnum::INDEX_FAISS_IVFSQ8H, (int32_t)EngineType::FAISS_IVFSQ8H},
{knowhere::IndexEnum::INDEX_NSG, (int32_t)EngineType::NSG_MIX},
#ifdef MILVUS_SUPPORT_SPTAG
{knowhere::IndexEnum::INDEX_SPTAG_KDT_RNT, EngineType::SPTAG_KDT},
{knowhere::IndexEnum::INDEX_SPTAG_BKT_RNT, EngineType::SPTAG_BKT},
{knowhere::IndexEnum::INDEX_SPTAG_KDT_RNT, (int32_t)EngineType::SPTAG_KDT},
{knowhere::IndexEnum::INDEX_SPTAG_BKT_RNT, (int32_t)EngineType::SPTAG_BKT},
#endif
{knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP, EngineType::FAISS_BIN_IDMAP},
{knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, EngineType::FAISS_BIN_IVFFLAT},
{knowhere::IndexEnum::INDEX_HNSW, EngineType::HNSW},
{knowhere::IndexEnum::INDEX_HNSW_SQ8NM, EngineType::HNSW_SQ8NM},
{knowhere::IndexEnum::INDEX_ANNOY, EngineType::ANNOY},
{knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP, (int32_t)EngineType::FAISS_BIN_IDMAP},
{knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, (int32_t)EngineType::FAISS_BIN_IVFFLAT},
{knowhere::IndexEnum::INDEX_HNSW, (int32_t)EngineType::HNSW},
{knowhere::IndexEnum::INDEX_HNSW_SQ8NM, (int32_t)EngineType::HNSW_SQ8NM},
{knowhere::IndexEnum::INDEX_ANNOY, (int32_t)EngineType::ANNOY},
};
static std::map<int32_t, std::string> s_index_type2name = {
{(int32_t)EngineType::FAISS_IDMAP, knowhere::IndexEnum::INDEX_FAISS_IDMAP},
{(int32_t)EngineType::FAISS_IVFFLAT, knowhere::IndexEnum::INDEX_FAISS_IVFFLAT},
{(int32_t)EngineType::FAISS_PQ, knowhere::IndexEnum::INDEX_FAISS_IVFPQ},
{(int32_t)EngineType::FAISS_IVFSQ8, knowhere::IndexEnum::INDEX_FAISS_IVFSQ8},
{(int32_t)EngineType::FAISS_IVFSQ8NR, knowhere::IndexEnum::INDEX_FAISS_IVFSQ8NR},
{(int32_t)EngineType::FAISS_IVFSQ8H, knowhere::IndexEnum::INDEX_FAISS_IVFSQ8H},
{(int32_t)EngineType::NSG_MIX, knowhere::IndexEnum::INDEX_NSG},
#ifdef MILVUS_SUPPORT_SPTAG
{(int32_t)EngineType::SPTAG_KDT, knowhere::IndexEnum::INDEX_SPTAG_KDT_RNT},
{(int32_t)EngineType::SPTAG_BKT, knowhere::IndexEnum::INDEX_SPTAG_BKT_RNT},
#endif
{(int32_t)EngineType::FAISS_BIN_IDMAP, knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP},
{(int32_t)EngineType::FAISS_BIN_IVFFLAT, knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT},
{(int32_t)EngineType::HNSW, knowhere::IndexEnum::INDEX_HNSW},
{(int32_t)EngineType::HNSW_SQ8NM, knowhere::IndexEnum::INDEX_HNSW_SQ8NM},
{(int32_t)EngineType::ANNOY, knowhere::IndexEnum::INDEX_ANNOY},
};
enum class MetricType {
......
core/src/db/meta/MySQLMetaImpl.cpp
浏览文件 @ c94699fb
......@@ -1010,8 +1010,8 @@ MySQLMetaImpl::UpdateCollectionIndex(const std::string& collection_id, const Col
statement << "UPDATE " << META_TABLES << " SET id = " << id << " ,state = " << state
<< " ,dimension = " << dimension << " ,created_on = " << created_on
<< " ,engine_type = " << index.engine_type_ << " ,index_params = " << mysqlpp::quote
<< index.extra_params_.dump() << " ,metric_type = " << index.metric_type_
<< " ,engine_type = " << index.index_name_ << " ,index_params = " << mysqlpp::quote
<< index.extra_params_.dump() << " ,metric_type = " << index.metric_name_
<< " WHERE table_id = " << mysqlpp::quote << collection_id << ";";
LOG_ENGINE_DEBUG_ << "UpdateCollectionIndex: " << statement.str();
......@@ -1386,11 +1386,11 @@ MySQLMetaImpl::DescribeCollectionIndex(const std::string& collection_id, Collect
if (res.num_rows() == 1) {
const mysqlpp::Row& resRow = res[0];
index.engine_type_ = resRow["engine_type"];
// index.index_name_ = resRow["engine_type"];
std::string str_index_params;
resRow["index_params"].to_string(str_index_params);
index.extra_params_ = milvus::json::parse(str_index_params);
index.metric_type_ = resRow["metric_type"];
// index.metric_name_ = resRow["metric_type"];
} else {
return Status(DB_NOT_FOUND, "Collection " + collection_id + " not found");
}
......
core/src/db/meta/SqliteMetaImpl.cpp
浏览文件 @ c94699fb
......@@ -784,9 +784,9 @@ SqliteMetaImpl::UpdateCollectionIndex(const std::string& collection_id, const Co
collection_schema.partition_tag_ = std::get<7>(collections[0]);
collection_schema.version_ = std::get<8>(collections[0]);
collection_schema.flush_lsn_ = std::get<9>(collections[0]);
collection_schema.engine_type_ = index.engine_type_;
// collection_schema.engine_type_ = index.engine_type_;
collection_schema.index_params_ = index.extra_params_.dump();
collection_schema.metric_type_ = index.metric_type_;
// collection_schema.metric_type_ = index.metric_type_;
ConnectorPtr->update(collection_schema);
} else {
......@@ -842,9 +842,9 @@ SqliteMetaImpl::DescribeCollectionIndex(const std::string& collection_id, Collec
c(&CollectionSchema::state_) != (int)CollectionSchema::TO_DELETE));
if (groups.size() == 1) {
index.engine_type_ = std::get<0>(groups[0]);
// index.engine_type_ = std::get<0>(groups[0]);
index.extra_params_ = milvus::json::parse(std::get<1>(groups[0]));
index.metric_type_ = std::get<2>(groups[0]);
// index.metric_type_ = std::get<2>(groups[0]);
} else {
return Status(DB_NOT_FOUND, "Collection " + collection_id + " not found");
}
......
core/src/grpc/gen-milvus/milvus.grpc.pb.cc
浏览文件 @ c94699fb
......@@ -38,8 +38,7 @@ static const char* MilvusService_method_names[] = {
"/milvus.grpc.MilvusService/GetEntityByID",
"/milvus.grpc.MilvusService/GetEntityIDs",
"/milvus.grpc.MilvusService/Search",
"/milvus.grpc.MilvusService/SearchByID",
"/milvus.grpc.MilvusService/SearchInFiles",
"/milvus.grpc.MilvusService/SearchInSegment",
"/milvus.grpc.MilvusService/Cmd",
"/milvus.grpc.MilvusService/DeleteByID",
"/milvus.grpc.MilvusService/PreloadCollection",
......@@ -74,15 +73,14 @@ MilvusService::Stub::Stub(const std::shared_ptr< ::grpc::ChannelInterface>& chan
, rpcmethod_GetEntityByID_(MilvusService_method_names[15], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_GetEntityIDs_(MilvusService_method_names[16], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_Search_(MilvusService_method_names[17], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_SearchByID_(MilvusService_method_names[18], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_SearchInFiles_(MilvusService_method_names[19], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_Cmd_(MilvusService_method_names[20], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_DeleteByID_(MilvusService_method_names[21], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_PreloadCollection_(MilvusService_method_names[22], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_ReloadSegments_(MilvusService_method_names[23], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_Flush_(MilvusService_method_names[24], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_Compact_(MilvusService_method_names[25], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_SearchPB_(MilvusService_method_names[26], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_SearchInSegment_(MilvusService_method_names[18], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_Cmd_(MilvusService_method_names[19], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_DeleteByID_(MilvusService_method_names[20], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_PreloadCollection_(MilvusService_method_names[21], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_ReloadSegments_(MilvusService_method_names[22], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_Flush_(MilvusService_method_names[23], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_Compact_(MilvusService_method_names[24], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
, rpcmethod_SearchPB_(MilvusService_method_names[25], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
{}
::grpc::Status MilvusService::Stub::CreateCollection(::grpc::ClientContext* context, const ::milvus::grpc::Mapping& request, ::milvus::grpc::Status* response) {
......@@ -309,11 +307,11 @@ void MilvusService::Stub::experimental_async::CreateIndex(::grpc::ClientContext*
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::Status>::Create(channel_.get(), cq, rpcmethod_CreateIndex_, context, request, false);
}
::grpc::Status MilvusService::Stub::DescribeIndex(::grpc::ClientContext* context, const ::milvus::grpc::CollectionName& request, ::milvus::grpc::IndexParam* response) {
::grpc::Status MilvusService::Stub::DescribeIndex(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam& request, ::milvus::grpc::IndexParam* response) {
return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_DescribeIndex_, context, request, response);
}
void MilvusService::Stub::experimental_async::DescribeIndex(::grpc::ClientContext* context, const ::milvus::grpc::CollectionName* request, ::milvus::grpc::IndexParam* response, std::function<void(::grpc::Status)> f) {
void MilvusService::Stub::experimental_async::DescribeIndex(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam* request, ::milvus::grpc::IndexParam* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_DescribeIndex_, context, request, response, std::move(f));
}
......@@ -321,7 +319,7 @@ void MilvusService::Stub::experimental_async::DescribeIndex(::grpc::ClientContex
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_DescribeIndex_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::DescribeIndex(::grpc::ClientContext* context, const ::milvus::grpc::CollectionName* request, ::milvus::grpc::IndexParam* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
void MilvusService::Stub::experimental_async::DescribeIndex(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam* request, ::milvus::grpc::IndexParam* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_DescribeIndex_, context, request, response, reactor);
}
......@@ -329,11 +327,11 @@ void MilvusService::Stub::experimental_async::DescribeIndex(::grpc::ClientContex
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_DescribeIndex_, context, request, response, reactor);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::IndexParam>* MilvusService::Stub::AsyncDescribeIndexRaw(::grpc::ClientContext* context, const ::milvus::grpc::CollectionName& request, ::grpc::CompletionQueue* cq) {
::grpc::ClientAsyncResponseReader< ::milvus::grpc::IndexParam>* MilvusService::Stub::AsyncDescribeIndexRaw(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::IndexParam>::Create(channel_.get(), cq, rpcmethod_DescribeIndex_, context, request, true);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::IndexParam>* MilvusService::Stub::PrepareAsyncDescribeIndexRaw(::grpc::ClientContext* context, const ::milvus::grpc::CollectionName& request, ::grpc::CompletionQueue* cq) {
::grpc::ClientAsyncResponseReader< ::milvus::grpc::IndexParam>* MilvusService::Stub::PrepareAsyncDescribeIndexRaw(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::IndexParam>::Create(channel_.get(), cq, rpcmethod_DescribeIndex_, context, request, false);
}
......@@ -589,60 +587,32 @@ void MilvusService::Stub::experimental_async::Search(::grpc::ClientContext* cont
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::QueryResult>::Create(channel_.get(), cq, rpcmethod_Search_, context, request, false);
}
::grpc::Status MilvusService::Stub::SearchByID(::grpc::ClientContext* context, const ::milvus::grpc::SearchByIDParam& request, ::milvus::grpc::QueryResult* response) {
return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_SearchByID_, context, request, response);
::grpc::Status MilvusService::Stub::SearchInSegment(::grpc::ClientContext* context, const ::milvus::grpc::SearchInSegmentParam& request, ::milvus::grpc::QueryResult* response) {
return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_SearchInSegment_, context, request, response);
}
void MilvusService::Stub::experimental_async::SearchByID(::grpc::ClientContext* context, const ::milvus::grpc::SearchByIDParam* request, ::milvus::grpc::QueryResult* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_SearchByID_, context, request, response, std::move(f));
void MilvusService::Stub::experimental_async::SearchInSegment(::grpc::ClientContext* context, const ::milvus::grpc::SearchInSegmentParam* request, ::milvus::grpc::QueryResult* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_SearchInSegment_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::SearchByID(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::QueryResult* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_SearchByID_, context, request, response, std::move(f));
void MilvusService::Stub::experimental_async::SearchInSegment(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::QueryResult* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_SearchInSegment_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::SearchByID(::grpc::ClientContext* context, const ::milvus::grpc::SearchByIDParam* request, ::milvus::grpc::QueryResult* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_SearchByID_, context, request, response, reactor);
void MilvusService::Stub::experimental_async::SearchInSegment(::grpc::ClientContext* context, const ::milvus::grpc::SearchInSegmentParam* request, ::milvus::grpc::QueryResult* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_SearchInSegment_, context, request, response, reactor);
}
void MilvusService::Stub::experimental_async::SearchByID(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::QueryResult* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_SearchByID_, context, request, response, reactor);
void MilvusService::Stub::experimental_async::SearchInSegment(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::QueryResult* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_SearchInSegment_, context, request, response, reactor);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::QueryResult>* MilvusService::Stub::AsyncSearchByIDRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchByIDParam& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::QueryResult>::Create(channel_.get(), cq, rpcmethod_SearchByID_, context, request, true);
::grpc::ClientAsyncResponseReader< ::milvus::grpc::QueryResult>* MilvusService::Stub::AsyncSearchInSegmentRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchInSegmentParam& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::QueryResult>::Create(channel_.get(), cq, rpcmethod_SearchInSegment_, context, request, true);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::QueryResult>* MilvusService::Stub::PrepareAsyncSearchByIDRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchByIDParam& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::QueryResult>::Create(channel_.get(), cq, rpcmethod_SearchByID_, context, request, false);
}
::grpc::Status MilvusService::Stub::SearchInFiles(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam& request, ::milvus::grpc::QueryResult* response) {
return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_SearchInFiles_, context, request, response);
}
void MilvusService::Stub::experimental_async::SearchInFiles(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam* request, ::milvus::grpc::QueryResult* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_SearchInFiles_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::SearchInFiles(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::QueryResult* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_SearchInFiles_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::SearchInFiles(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam* request, ::milvus::grpc::QueryResult* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_SearchInFiles_, context, request, response, reactor);
}
void MilvusService::Stub::experimental_async::SearchInFiles(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::QueryResult* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_SearchInFiles_, context, request, response, reactor);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::QueryResult>* MilvusService::Stub::AsyncSearchInFilesRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::QueryResult>::Create(channel_.get(), cq, rpcmethod_SearchInFiles_, context, request, true);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::QueryResult>* MilvusService::Stub::PrepareAsyncSearchInFilesRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::QueryResult>::Create(channel_.get(), cq, rpcmethod_SearchInFiles_, context, request, false);
::grpc::ClientAsyncResponseReader< ::milvus::grpc::QueryResult>* MilvusService::Stub::PrepareAsyncSearchInSegmentRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchInSegmentParam& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::QueryResult>::Create(channel_.get(), cq, rpcmethod_SearchInSegment_, context, request, false);
}
::grpc::Status MilvusService::Stub::Cmd(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::milvus::grpc::StringReply* response) {
......@@ -885,7 +855,7 @@ MilvusService::Service::Service() {
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[8],
::grpc::internal::RpcMethod::NORMAL_RPC,
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::CollectionName, ::milvus::grpc::IndexParam>(
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::IndexParam, ::milvus::grpc::IndexParam>(
std::mem_fn(&MilvusService::Service::DescribeIndex), this)));
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[9],
......@@ -935,45 +905,40 @@ MilvusService::Service::Service() {
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[18],
::grpc::internal::RpcMethod::NORMAL_RPC,
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::SearchByIDParam, ::milvus::grpc::QueryResult>(
std::mem_fn(&MilvusService::Service::SearchByID), this)));
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::SearchInSegmentParam, ::milvus::grpc::QueryResult>(
std::mem_fn(&MilvusService::Service::SearchInSegment), this)));
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[19],
::grpc::internal::RpcMethod::NORMAL_RPC,
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::SearchInFilesParam, ::milvus::grpc::QueryResult>(
std::mem_fn(&MilvusService::Service::SearchInFiles), this)));
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[20],
::grpc::internal::RpcMethod::NORMAL_RPC,
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::Command, ::milvus::grpc::StringReply>(
std::mem_fn(&MilvusService::Service::Cmd), this)));
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[21],
MilvusService_method_names[20],
::grpc::internal::RpcMethod::NORMAL_RPC,
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::DeleteByIDParam, ::milvus::grpc::Status>(
std::mem_fn(&MilvusService::Service::DeleteByID), this)));
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[22],
MilvusService_method_names[21],
::grpc::internal::RpcMethod::NORMAL_RPC,
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::CollectionName, ::milvus::grpc::Status>(
std::mem_fn(&MilvusService::Service::PreloadCollection), this)));
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[23],
MilvusService_method_names[22],
::grpc::internal::RpcMethod::NORMAL_RPC,
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::ReLoadSegmentsParam, ::milvus::grpc::Status>(
std::mem_fn(&MilvusService::Service::ReloadSegments), this)));
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[24],
MilvusService_method_names[23],
::grpc::internal::RpcMethod::NORMAL_RPC,
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::FlushParam, ::milvus::grpc::Status>(
std::mem_fn(&MilvusService::Service::Flush), this)));
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[25],
MilvusService_method_names[24],
::grpc::internal::RpcMethod::NORMAL_RPC,
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::CollectionName, ::milvus::grpc::Status>(
std::mem_fn(&MilvusService::Service::Compact), this)));
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[26],
MilvusService_method_names[25],
::grpc::internal::RpcMethod::NORMAL_RPC,
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::SearchParamPB, ::milvus::grpc::QueryResult>(
std::mem_fn(&MilvusService::Service::SearchPB), this)));
......@@ -1038,7 +1003,7 @@ MilvusService::Service::~Service() {
return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
}
::grpc::Status MilvusService::Service::DescribeIndex(::grpc::ServerContext* context, const ::milvus::grpc::CollectionName* request, ::milvus::grpc::IndexParam* response) {
::grpc::Status MilvusService::Service::DescribeIndex(::grpc::ServerContext* context, const ::milvus::grpc::IndexParam* request, ::milvus::grpc::IndexParam* response) {
(void) context;
(void) request;
(void) response;
......@@ -1108,14 +1073,7 @@ MilvusService::Service::~Service() {
return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
}
::grpc::Status MilvusService::Service::SearchByID(::grpc::ServerContext* context, const ::milvus::grpc::SearchByIDParam* request, ::milvus::grpc::QueryResult* response) {
(void) context;
(void) request;
(void) response;
return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
}
::grpc::Status MilvusService::Service::SearchInFiles(::grpc::ServerContext* context, const ::milvus::grpc::SearchInFilesParam* request, ::milvus::grpc::QueryResult* response) {
::grpc::Status MilvusService::Service::SearchInSegment(::grpc::ServerContext* context, const ::milvus::grpc::SearchInSegmentParam* request, ::milvus::grpc::QueryResult* response) {
(void) context;
(void) request;
(void) response;
......
core/src/grpc/milvus.proto
浏览文件 @ c94699fb
......@@ -48,6 +48,14 @@ message CollectionNameList {
repeated string collection_names = 2;
}
/**
* @brief Field name
*/
message FieldName {
string collection_name = 1;
string field_name = 2;
}
/**
* @brief Collection mapping
*/
......@@ -158,22 +166,11 @@ message SearchParam {
/**
* @brief Params for searching vector in files
*/
message SearchInFilesParam {
message SearchInSegmentParam {
repeated string file_id_array = 1;
SearchParam search_param = 2;
}
/**
* @brief Params for searching vector by ID
*/
message SearchByIDParam {
string collection_name = 1;
repeated string partition_tag_array = 2;
repeated int64 id_array = 3;
int64 topk = 4;
repeated KeyValuePair extra_params = 5;
}
/**
* @brief Params for reloading segments
*/
......@@ -237,7 +234,18 @@ message Command {
/**
* @brief Index params
* @index_type: 0-invalid, 1-idmap, 2-ivflat, 3-ivfsq8, 4-nsgmix
* @collection_name: target collection
* @field_name: target field
* @index_name: a name for index provided by user, unique within this field
* @extra_params: index parameters in json format
* for vector field:
* extra_params["index_type"] = one of the values: IDMAP, IVFLAT, IVFSQ8, NSGMIX, IVFSQ8H,
* PQ, HNSW, HNSW_SQ8NM, ANNOY
* extra_params["metric_type"] = one of the values: L2, IP, HAMMING, JACCARD, TANIMOTO
* SUBSTRUCTURE, SUPERSTRUCTURE
* extra_params["params"] = extra parameters for index, for example ivflat: {nlist: 2048}
* for structured field:
* extra_params["index_type"] = one of the values: SORTED
*/
message IndexParam {
Status status = 1;
......@@ -440,7 +448,7 @@ service MilvusService {
*
* @param CollectionName, collection name is going to be deleted.
*
* @return CollectionNameList
* @return Status
*/
rpc DropCollection(CollectionName) returns (Status) {}
......@@ -456,16 +464,16 @@ service MilvusService {
/**
* @brief This method is used to describe index
*
* @param CollectionName, target collection name.
* @param IndexParam, target index.
*
* @return IndexParam
*/
rpc DescribeIndex(CollectionName) returns (IndexParam) {}
rpc DescribeIndex(IndexParam) returns (IndexParam) {}
/**
* @brief This method is used to drop index
*
* @param CollectionName, target collection name.
* @param IndexParam, target field. if the IndexParam.field_name is empty, will drop all index of the collection
*
* @return Status
*/
......@@ -543,23 +551,14 @@ service MilvusService {
*/
rpc Search(SearchParam) returns (QueryResult) {}
/**
* @brief This method is used to query vector by id.
*
* @param SearchByIDParam, search parameters.
*
* @return TopKQueryResult
*/
rpc SearchByID(SearchByIDParam) returns (QueryResult) {}
/**
* @brief This method is used to query vector in specified files.
*
* @param SearchInFilesParam, search in files paremeters.
* @param SearchInSegmentParam, target segments to search.
*
* @return TopKQueryResult
*/
rpc SearchInFiles(SearchInFilesParam) returns (QueryResult) {}
rpc SearchInSegment(SearchInSegmentParam) returns (QueryResult) {}
/**
* @brief This method is used to give the server status.
......
core/src/server/ValidationUtil.cpp
浏览文件 @ c94699fb
......@@ -227,128 +227,97 @@ ValidateTableDimension(int64_t dimension, int64_t metric_type) {
}
Status
ValidateCollectionIndexType(int32_t index_type) {
int engine_type = static_cast<int>(engine::EngineType(index_type));
if (engine_type <= 0 || engine_type > static_cast<int>(engine::EngineType::MAX_VALUE)) {
std::string index_type_str;
for (auto it = engine::s_map_engine_type.begin(); it != engine::s_map_engine_type.end(); it++) {
if (it->second == (engine::EngineType)index_type) {
index_type_str = it->first;
}
}
std::string msg =
"Invalid index type: " + index_type_str + ". " + "Make sure the index type is in IndexType list.";
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_INDEX_TYPE, msg);
}
#ifndef MILVUS_GPU_VERSION
// special case, hybird index only available in customize faiss library
if (engine_type == static_cast<int>(engine::EngineType::FAISS_IVFSQ8H)) {
std::string msg = "Unsupported index type: " + std::to_string(index_type);
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_INDEX_TYPE, msg);
}
#endif
ValidateIndexType(const std::string& index_type) {
return Status::OK();
}
Status
ValidateMetricType(const std::string& metric_type) {
return Status::OK();
}
Status
ValidateIndexParams(const milvus::json& index_params, int64_t dimension, int32_t index_type) {
switch (index_type) {
case (int32_t)engine::EngineType::FAISS_IDMAP:
case (int32_t)engine::EngineType::FAISS_BIN_IDMAP: {
break;
ValidateIndexParams(const milvus::json& index_params, int64_t dimension, const std::string& index_type) {
if (index_type == knowhere::IndexEnum::INDEX_FAISS_IDMAP ||
index_type == knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP) {
return Status::OK();
} else if (index_type == knowhere::IndexEnum::INDEX_FAISS_IVFFLAT ||
index_type == knowhere::IndexEnum::INDEX_FAISS_IVFSQ8 ||
index_type == knowhere::IndexEnum::INDEX_FAISS_IVFSQ8NR ||
index_type == knowhere::IndexEnum::INDEX_FAISS_IVFSQ8H ||
index_type == knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT) {
auto status = CheckParameterRange(index_params, knowhere::IndexParams::nlist, 1, 999999);
if (!status.ok()) {
return status;
}
case (int32_t)engine::EngineType::FAISS_IVFFLAT:
case (int32_t)engine::EngineType::FAISS_IVFSQ8:
case (int32_t)engine::EngineType::FAISS_IVFSQ8NR:
case (int32_t)engine::EngineType::FAISS_IVFSQ8H:
case (int32_t)engine::EngineType::FAISS_BIN_IVFFLAT: {
auto status = CheckParameterRange(index_params, knowhere::IndexParams::nlist, 1, 999999);
if (!status.ok()) {
return status;
}
break;
} else if (index_type == knowhere::IndexEnum::INDEX_FAISS_IVFPQ) {
auto status = CheckParameterRange(index_params, knowhere::IndexParams::nlist, 1, 999999);
if (!status.ok()) {
return status;
}
case (int32_t)engine::EngineType::FAISS_PQ: {
auto status = CheckParameterRange(index_params, knowhere::IndexParams::nlist, 1, 999999);
if (!status.ok()) {
return status;
}
status = CheckParameterExistence(index_params, knowhere::IndexParams::m);
if (!status.ok()) {
return status;
}
status = CheckParameterExistence(index_params, knowhere::IndexParams::m);
if (!status.ok()) {
return status;
}
// special check for 'm' parameter
std::vector<int64_t> resset;
milvus::knowhere::IVFPQConfAdapter::GetValidMList(dimension, resset);
int64_t m_value = index_params[knowhere::IndexParams::m];
if (resset.empty()) {
std::string msg = "Invalid collection dimension, unable to get reasonable values for 'm'";
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_COLLECTION_DIMENSION, msg);
}
// special check for 'm' parameter
std::vector<int64_t> resset;
milvus::knowhere::IVFPQConfAdapter::GetValidMList(dimension, resset);
int64_t m_value = index_params[knowhere::IndexParams::m];
if (resset.empty()) {
std::string msg = "Invalid collection dimension, unable to get reasonable values for 'm'";
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_COLLECTION_DIMENSION, msg);
}
auto iter = std::find(std::begin(resset), std::end(resset), m_value);
if (iter == std::end(resset)) {
std::string msg =
"Invalid " + std::string(knowhere::IndexParams::m) + ", must be one of the following values: ";
for (size_t i = 0; i < resset.size(); i++) {
if (i != 0) {
msg += ",";
}
msg += std::to_string(resset[i]);
auto iter = std::find(std::begin(resset), std::end(resset), m_value);
if (iter == std::end(resset)) {
std::string msg =
"Invalid " + std::string(knowhere::IndexParams::m) + ", must be one of the following values: ";
for (size_t i = 0; i < resset.size(); i++) {
if (i != 0) {
msg += ",";
}
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_ARGUMENT, msg);
msg += std::to_string(resset[i]);
}
break;
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_ARGUMENT, msg);
}
case (int32_t)engine::EngineType::NSG_MIX: {
auto status = CheckParameterRange(index_params, knowhere::IndexParams::search_length, 10, 300);
if (!status.ok()) {
return status;
}
status = CheckParameterRange(index_params, knowhere::IndexParams::out_degree, 5, 300);
if (!status.ok()) {
return status;
}
status = CheckParameterRange(index_params, knowhere::IndexParams::candidate, 50, 1000);
if (!status.ok()) {
return status;
}
status = CheckParameterRange(index_params, knowhere::IndexParams::knng, 5, 300);
if (!status.ok()) {
return status;
}
break;
} else if (index_type == knowhere::IndexEnum::INDEX_NSG) {
auto status = CheckParameterRange(index_params, knowhere::IndexParams::search_length, 10, 300);
if (!status.ok()) {
return status;
}
case (int32_t)engine::EngineType::HNSW_SQ8NM:
case (int32_t)engine::EngineType::HNSW: {
auto status = CheckParameterRange(index_params, knowhere::IndexParams::M, 4, 64);
if (!status.ok()) {
return status;
}
status = CheckParameterRange(index_params, knowhere::IndexParams::efConstruction, 8, 512);
if (!status.ok()) {
return status;
}
break;
status = CheckParameterRange(index_params, knowhere::IndexParams::out_degree, 5, 300);
if (!status.ok()) {
return status;
}
case (int32_t)engine::EngineType::ANNOY: {
auto status = CheckParameterRange(index_params, knowhere::IndexParams::n_trees, 1, 1024);
if (!status.ok()) {
return status;
}
break;
status = CheckParameterRange(index_params, knowhere::IndexParams::candidate, 50, 1000);
if (!status.ok()) {
return status;
}
status = CheckParameterRange(index_params, knowhere::IndexParams::knng, 5, 300);
if (!status.ok()) {
return status;
}
} else if (index_type == knowhere::IndexEnum::INDEX_HNSW || index_type == knowhere::IndexEnum::INDEX_HNSW_SQ8NM) {
auto status = CheckParameterRange(index_params, knowhere::IndexParams::M, 4, 64);
if (!status.ok()) {
return status;
}
status = CheckParameterRange(index_params, knowhere::IndexParams::efConstruction, 8, 512);
if (!status.ok()) {
return status;
}
} else if (index_type == knowhere::IndexEnum::INDEX_ANNOY) {
auto status = CheckParameterRange(index_params, knowhere::IndexParams::n_trees, 1, 1024);
if (!status.ok()) {
return status;
}
}
return Status::OK();
}
......
core/src/server/ValidationUtil.h
浏览文件 @ c94699fb
......@@ -37,10 +37,13 @@ extern Status
ValidateTableDimension(int64_t dimension, int64_t metric_type);
extern Status
ValidateCollectionIndexType(int32_t index_type);
ValidateIndexType(const std::string& index_type);
extern Status
ValidateIndexParams(const milvus::json& index_params, int64_t dimension, int32_t index_type);
ValidateMetricType(const std::string& metric_type);
extern Status
ValidateIndexParams(const milvus::json& index_params, int64_t dimension, const std::string& index_type);
extern Status
ValidateSearchParams(const milvus::json& search_params, const engine::meta::CollectionSchema& collection_schema,
......@@ -55,9 +58,6 @@ ValidateVectorDataSize(const engine::VectorsData& vectors, const engine::meta::C
extern Status
ValidateCollectionIndexFileSize(int64_t index_file_size);
extern Status
ValidateCollectionIndexMetricType(int32_t metric_type);
extern Status
ValidateSearchTopk(int64_t top_k);
......
core/src/server/delivery/request/CreateIndexRequest.cpp
浏览文件 @ c94699fb
......@@ -10,6 +10,7 @@
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "server/delivery/request/CreateIndexRequest.h"
#include "db/SnapshotUtils.h"
#include "db/Utils.h"
#include "server/DBWrapper.h"
#include "server/ValidationUtil.h"
......@@ -69,9 +70,6 @@ CreateIndexRequest::OnExecute() {
engine::snapshot::CollectionPtr collection;
engine::snapshot::CollectionMappings fields_schema;
status = DBWrapper::SSDB()->DescribeCollection(collection_name_, collection, fields_schema);
fiu_do_on("CreateIndexRequest.OnExecute.not_has_collection",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
fiu_do_on("CreateIndexRequest.OnExecute.throw_std.exception", throw std::exception());
if (!status.ok()) {
if (status.code() == DB_NOT_FOUND) {
return Status(SERVER_COLLECTION_NOT_EXIST, CollectionNotExistMsg(collection_name_));
......@@ -80,64 +78,60 @@ CreateIndexRequest::OnExecute() {
}
}
int64_t dimension;
// pick up field
engine::snapshot::FieldPtr field;
for (auto field_it = fields_schema.begin(); field_it != fields_schema.end(); field_it++) {
auto type = field_it->first->GetFtype();
if (type == (int64_t)engine::meta::hybrid::DataType::VECTOR_FLOAT ||
type == (int64_t)engine::meta::hybrid::DataType::VECTOR_BINARY) {
auto params = field_it->first->GetParams();
dimension = params[engine::DIMENSION].get<int64_t>();
if (field_it->first->GetName() == field_name_) {
field = field_it->first;
break;
}
}
int32_t index_type = 0;
if (json_params_.contains("index_type")) {
auto index_type_str = json_params_["index_type"].get<std::string>();
if (engine::s_map_engine_type.find(index_type_str) == engine::s_map_engine_type.end()) {
return Status(SERVER_INVALID_ARGUMENT, "Set wrong index type");
}
index_type = (int32_t)engine::s_map_engine_type.at(index_type_str);
if (field == nullptr) {
return Status(SERVER_INVALID_FIELD_NAME, "Invalid field name");
}
status = ValidateCollectionIndexType(index_type);
if (!status.ok()) {
return status;
}
engine::CollectionIndex index;
if (engine::IsVectorField(field)) {
int32_t field_type = field->GetFtype();
auto params = field->GetParams();
int64_t dimension = params[engine::DIMENSION].get<int64_t>();
// validate index type
std::string index_type = 0;
if (json_params_.contains("index_type")) {
index_type = json_params_["index_type"].get<std::string>();
}
status = ValidateIndexType(index_type);
if (!status.ok()) {
return status;
}
status = ValidateIndexParams(json_params_, dimension, index_type);
if (!status.ok()) {
return status;
}
// validate metric type
std::string metric_type = 0;
if (json_params_.contains("metric_type")) {
metric_type = json_params_["metric_type"].get<std::string>();
}
status = ValidateMetricType(metric_type);
if (!status.ok()) {
return status;
}
// step 2: binary and float vector support different index/metric type, need to adapt here
engine::meta::CollectionSchema collection_info;
collection_info.collection_id_ = collection_name_;
status = DBWrapper::DB()->DescribeCollection(collection_info);
int32_t adapter_index_type = index_type;
if (engine::utils::IsBinaryMetricType(collection_info.metric_type_)) { // binary vector not allow
if (adapter_index_type == static_cast<int32_t>(engine::EngineType::FAISS_IDMAP)) {
adapter_index_type = static_cast<int32_t>(engine::EngineType::FAISS_BIN_IDMAP);
} else if (adapter_index_type == static_cast<int32_t>(engine::EngineType::FAISS_IVFFLAT)) {
adapter_index_type = static_cast<int32_t>(engine::EngineType::FAISS_BIN_IVFFLAT);
// validate index parameters
status = ValidateIndexParams(json_params_, dimension, index_type);
if (!status.ok()) {
return status;
}
}
rc.RecordSection("check validation");
rc.RecordSection("check validation");
// step 3: create index
engine::CollectionIndex index;
if (json_params_.contains("metric_type")) {
index.metric_type_ = (int32_t)engine::s_map_metric_type.at(json_params_["metric_type"]);
index.index_name_ = index_name_;
index.metric_name_ = metric_type;
index.extra_params_ = json_params_;
} else {
index.index_name_ = index_name_;
}
index.engine_type_ = adapter_index_type;
index.index_name_ = index_name_;
index.extra_params_ = json_params_;
status = DBWrapper::SSDB()->CreateIndex(context_, collection_name_, field_name_, index);
fiu_do_on("CreateIndexRequest.OnExecute.create_index_fail",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
if (!status.ok()) {
return status;
}
......
core/src/server/delivery/request/DescribeIndexRequest.cpp
浏览文件 @ c94699fb
......@@ -61,35 +61,47 @@ DescribeIndexRequest::OnExecute() {
}
}
// TODO(yukun): Currently field name is vector field name
std::string field_name;
for (auto field_it = fields_schema.begin(); field_it != fields_schema.end(); field_it++) {
auto type = field_it->first->GetFtype();
if (type == (int64_t)engine::meta::hybrid::DataType::VECTOR_FLOAT ||
type == (int64_t)engine::meta::hybrid::DataType::VECTOR_BINARY) {
field_name = field_it->first->GetName();
break;
}
}
// step 2: check collection existence
engine::CollectionIndex index;
status = DBWrapper::SSDB()->DescribeIndex(collection_name_, field_name, index);
if (!status.ok()) {
return status;
}
// for binary vector, IDMAP and IVFLAT will be treated as BIN_IDMAP and BIN_IVFLAT internally
// return IDMAP and IVFLAT for outside caller
if (index.engine_type_ == (int32_t)engine::EngineType::FAISS_BIN_IDMAP) {
index.engine_type_ = (int32_t)engine::EngineType::FAISS_IDMAP;
} else if (index.engine_type_ == (int32_t)engine::EngineType::FAISS_BIN_IVFFLAT) {
index.engine_type_ = (int32_t)engine::EngineType::FAISS_IVFFLAT;
}
index_param_.collection_name_ = collection_name_;
index_param_.index_type_ = index.engine_type_;
index_param_.extra_params_ = index.extra_params_.dump();
// // pick up field
// engine::snapshot::FieldPtr field;
// for (auto field_it = fields_schema.begin(); field_it != fields_schema.end(); field_it++) {
// if (field_it->first->GetName() == field_name_) {
// field = field_it->first;
// break;
// }
// }
// if (field == nullptr) {
// return Status(SERVER_INVALID_FIELD_NAME, "Invalid field name");
// }
//
// // TODO(yukun): Currently field name is vector field name
// std::string field_name;
// for (auto field_it = fields_schema.begin(); field_it != fields_schema.end(); field_it++) {
// auto type = field_it->first->GetFtype();
// if (type == (int64_t)engine::meta::hybrid::DataType::VECTOR_FLOAT ||
// type == (int64_t)engine::meta::hybrid::DataType::VECTOR_BINARY) {
// field_name = field_it->first->GetName();
// break;
// }
// }
//
// // step 2: check collection existence
// engine::CollectionIndex index;
// status = DBWrapper::SSDB()->DescribeIndex(collection_name_, field_name, index);
// if (!status.ok()) {
// return status;
// }
//
// // for binary vector, IDMAP and IVFLAT will be treated as BIN_IDMAP and BIN_IVFLAT internally
// // return IDMAP and IVFLAT for outside caller
// if (index.engine_type_ == (int32_t)engine::EngineType::FAISS_BIN_IDMAP) {
// index.engine_type_ = (int32_t)engine::EngineType::FAISS_IDMAP;
// } else if (index.engine_type_ == (int32_t)engine::EngineType::FAISS_BIN_IVFFLAT) {
// index.engine_type_ = (int32_t)engine::EngineType::FAISS_IVFFLAT;
// }
//
// index_param_.collection_name_ = collection_name_;
// index_param_.index_type_ = index.engine_type_;
// index_param_.extra_params_ = index.extra_params_.dump();
} catch (std::exception& ex) {
return Status(SERVER_UNEXPECTED_ERROR, ex.what());
}
......
core/src/server/grpc_impl/GrpcRequestHandler.cpp
浏览文件 @ c94699fb
......@@ -828,48 +828,8 @@ GrpcRequestHandler::GetEntityIDs(::grpc::ServerContext* context, const ::milvus:
//}
::grpc::Status
GrpcRequestHandler::SearchByID(::grpc::ServerContext* context, const ::milvus::grpc::SearchByIDParam* request,
::milvus::grpc::QueryResult* response) {
// CHECK_NULLPTR_RETURN(request);
// LOG_SERVER_INFO_ << LogOut("Request [%s] %s begin.", GetContext(context)->RequestID().c_str(), __func__);
//
// // step 1: partition tags
// std::vector<std::string> partitions;
// std::copy(request->partition_tag_array().begin(), request->partition_tag_array().end(),
// std::back_inserter(partitions));
//
// // step 2: partition tags
// std::vector<int64_t> id_array;
// for (int i = 0; i < request->id_array_size(); i++) {
// id_array.push_back(request->id_array(i));
// }
//
// // step 3: parse extra parameters
// milvus::json json_params;
// for (int i = 0; i < request->extra_params_size(); i++) {
// const ::milvus::grpc::KeyValuePair& extra = request->extra_params(i);
// if (extra.key() == EXTRA_PARAM_KEY) {
// json_params = json::parse(extra.value());
// }
// }
//
// // step 4: search vectors
// TopKQueryResult result;
// Status status = request_handler_.SearchByID(GetContext(context), request->collection_name(), id_array,
// request->topk(), json_params, partitions, result);
//
// // step 5: construct and return result
// ConstructResults(result, response);
//
// LOG_SERVER_INFO_ << LogOut("Request [%s] %s end.", GetContext(context)->RequestID().c_str(), __func__);
// SET_RESPONSE(response->mutable_status(), status, context);
//
// return ::grpc::Status::OK;
}
::grpc::Status
GrpcRequestHandler::SearchInFiles(::grpc::ServerContext* context, const ::milvus::grpc::SearchInFilesParam* request,
::milvus::grpc::QueryResult* response) {
GrpcRequestHandler::SearchInSegment(::grpc::ServerContext* context, const ::milvus::grpc::SearchInSegmentParam* request,
::milvus::grpc::QueryResult* response) {
// CHECK_NULLPTR_RETURN(request);
// LOG_SERVER_INFO_ << LogOut("Request [%s] %s begin.", GetContext(context)->RequestID().c_str(), __func__);
//
......@@ -1098,7 +1058,7 @@ GrpcRequestHandler::ReloadSegments(::grpc::ServerContext* context, const ::milvu
}
::grpc::Status
GrpcRequestHandler::DescribeIndex(::grpc::ServerContext* context, const ::milvus::grpc::CollectionName* request,
GrpcRequestHandler::DescribeIndex(::grpc::ServerContext* context, const ::milvus::grpc::IndexParam* request,
::milvus::grpc::IndexParam* response) {
CHECK_NULLPTR_RETURN(request);
LOG_SERVER_INFO_ << LogOut("Request [%s] %s begin.", GetContext(context)->RequestID().c_str(), __func__);
......
core/src/server/grpc_impl/GrpcRequestHandler.h
浏览文件 @ c94699fb
......@@ -160,7 +160,7 @@ class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service,
//
// @return IndexParam
::grpc::Status
DescribeIndex(::grpc::ServerContext* context, const ::milvus::grpc::CollectionName* request,
DescribeIndex(::grpc::ServerContext* context, const ::milvus::grpc::IndexParam* request,
::milvus::grpc::IndexParam* response) override;
// *
// @brief This method is used to drop index
......@@ -247,16 +247,6 @@ class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service,
Search(::grpc::ServerContext* context, const ::milvus::grpc::SearchParam* request,
::milvus::grpc::QueryResult* response) override;
// *
// @brief This method is used to query vector by id.
//
// @param SearchByIDParam, search parameters.
//
// @return TopKQueryResult
::grpc::Status
SearchByID(::grpc::ServerContext* context, const ::milvus::grpc::SearchByIDParam* request,
::milvus::grpc::QueryResult* response);
// *
// @brief This method is used to query vector in specified files.
//
......@@ -264,8 +254,8 @@ class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service,
//
// @return TopKQueryResultList
::grpc::Status
SearchInFiles(::grpc::ServerContext* context, const ::milvus::grpc::SearchInFilesParam* request,
::milvus::grpc::QueryResult* response) override;
SearchInSegment(::grpc::ServerContext* context, const ::milvus::grpc::SearchInSegmentParam* request,
::milvus::grpc::QueryResult* response) override;
// *
// @brief This method is used to give the server status.
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或