提交 eea09cd9 编写于 作者: P peng.xu

Merge branch 'branch-0.4.0' into 'branch-0.4.0'

MS-339 implement new api

See merge request megasearch/milvus!400

Former-commit-id: 4724209a0e5e3d144e313e8ecf6499e9fc3e15c4
......@@ -41,6 +41,9 @@ Please mark all change in change log and use the ticket from JIRA.
## New Feature
- MS-343 - Implement ResourceMgr
- MS-338 - NewAPI: refine code to support CreateIndex
- MS-339 - NewAPI: refine code to support DropIndex
- MS-340 - NewAPI: implement DescribeIndex
## Task
- MS-297 - disable mysql unit test
......
......@@ -23,9 +23,9 @@ typedef std::vector<QueryResult> QueryResults;
struct TableIndex {
int32_t engine_type_ = (int)EngineType::FAISS_IDMAP;
int32_t nlist = 16384;
int32_t index_file_size = 1024; //MB
int32_t metric_type = (int)MetricType::L2;
int32_t nlist_ = 16384;
int32_t index_file_size_ = 1024; //MB
int32_t metric_type_ = (int)MetricType::L2;
};
} // namespace engine
......
......@@ -148,9 +148,9 @@ Status DeleteTableFilePath(const DBMetaOptions& options, meta::TableFileSchema&
bool IsSameIndex(const TableIndex& index1, const TableIndex& index2) {
return index1.engine_type_ == index2.engine_type_
&& index1.nlist == index2.nlist
&& index1.index_file_size == index2.index_file_size
&& index1.metric_type == index2.metric_type;
&& index1.nlist_ == index2.nlist_
&& index1.index_file_size_ == index2.index_file_size_
&& index1.metric_type_ == index2.metric_type_;
}
} // namespace utils
......
此差异已折叠。
......@@ -367,9 +367,9 @@ Status SqliteMetaImpl::UpdateTableIndexParam(const std::string &table_id, const
table_schema.dimension_ = std::get<2>(tables[0]);
table_schema.created_on_ = std::get<3>(tables[0]);
table_schema.engine_type_ = index.engine_type_;
table_schema.nlist_ = index.nlist;
table_schema.index_file_size_ = index.index_file_size;
table_schema.metric_type_ = index.metric_type;
table_schema.nlist_ = index.nlist_;
table_schema.index_file_size_ = index.index_file_size_;
table_schema.metric_type_ = index.metric_type_;
ConnectorPtr->update(table_schema);
} else {
......@@ -407,9 +407,9 @@ Status SqliteMetaImpl::DescribeTableIndex(const std::string &table_id, TableInde
if (groups.size() == 1) {
index.engine_type_ = std::get<0>(groups[0]);
index.nlist = std::get<1>(groups[0]);
index.index_file_size = std::get<2>(groups[0]);
index.metric_type = std::get<3>(groups[0]);
index.nlist_ = std::get<1>(groups[0]);
index.index_file_size_ = std::get<2>(groups[0]);
index.metric_type_ = std::get<3>(groups[0]);
} else {
return Status::NotFound("Table " + table_id + " not found");
}
......
......@@ -545,7 +545,7 @@ const char descriptor_table_protodef_milvus_2eproto[] PROTOBUF_SECTION_VARIABLE(
"_reply\030\002 \001(\010\"M\n\rTableRowCount\022#\n\006status\030"
"\001 \001(\0132\023.milvus.grpc.Status\022\027\n\017table_row_"
"count\030\002 \001(\003\"\026\n\007Command\022\013\n\003cmd\030\001 \001(\t\"X\n\005I"
"ndex\022\022\n\nindex_type\030\001 \001(\005\022\r\n\005nlist\030\002 \001(\003\022"
"ndex\022\022\n\nindex_type\030\001 \001(\005\022\r\n\005nlist\030\002 \001(\005\022"
"\027\n\017index_file_size\030\003 \001(\005\022\023\n\013metric_type\030"
"\004 \001(\005\"[\n\nIndexParam\022*\n\ntable_name\030\001 \001(\0132"
"\026.milvus.grpc.TableName\022!\n\005index\030\002 \001(\0132\022"
......@@ -5268,16 +5268,16 @@ Index::Index(const Index& from)
: ::PROTOBUF_NAMESPACE_ID::Message(),
_internal_metadata_(nullptr) {
_internal_metadata_.MergeFrom(from._internal_metadata_);
::memcpy(&nlist_, &from.nlist_,
::memcpy(&index_type_, &from.index_type_,
static_cast<size_t>(reinterpret_cast<char*>(&metric_type_) -
reinterpret_cast<char*>(&nlist_)) + sizeof(metric_type_));
reinterpret_cast<char*>(&index_type_)) + sizeof(metric_type_));
// @@protoc_insertion_point(copy_constructor:milvus.grpc.Index)
}
void Index::SharedCtor() {
::memset(&nlist_, 0, static_cast<size_t>(
::memset(&index_type_, 0, static_cast<size_t>(
reinterpret_cast<char*>(&metric_type_) -
reinterpret_cast<char*>(&nlist_)) + sizeof(metric_type_));
reinterpret_cast<char*>(&index_type_)) + sizeof(metric_type_));
}
Index::~Index() {
......@@ -5303,9 +5303,9 @@ void Index::Clear() {
// Prevent compiler warnings about cached_has_bits being unused
(void) cached_has_bits;
::memset(&nlist_, 0, static_cast<size_t>(
::memset(&index_type_, 0, static_cast<size_t>(
reinterpret_cast<char*>(&metric_type_) -
reinterpret_cast<char*>(&nlist_)) + sizeof(metric_type_));
reinterpret_cast<char*>(&index_type_)) + sizeof(metric_type_));
_internal_metadata_.Clear();
}
......@@ -5324,7 +5324,7 @@ const char* Index::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::inte
CHK_(ptr);
} else goto handle_unusual;
continue;
// int64 nlist = 2;
// int32 nlist = 2;
case 2:
if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 16)) {
nlist_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr);
......@@ -5388,12 +5388,12 @@ bool Index::MergePartialFromCodedStream(
break;
}
// int64 nlist = 2;
// int32 nlist = 2;
case 2: {
if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (16 & 0xFF)) {
DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive<
::PROTOBUF_NAMESPACE_ID::int64, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT64>(
::PROTOBUF_NAMESPACE_ID::int32, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT32>(
input, &nlist_)));
} else {
goto handle_unusual;
......@@ -5459,9 +5459,9 @@ void Index::SerializeWithCachedSizes(
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32(1, this->index_type(), output);
}
// int64 nlist = 2;
// int32 nlist = 2;
if (this->nlist() != 0) {
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64(2, this->nlist(), output);
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32(2, this->nlist(), output);
}
// int32 index_file_size = 3;
......@@ -5492,9 +5492,9 @@ void Index::SerializeWithCachedSizes(
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(1, this->index_type(), target);
}
// int64 nlist = 2;
// int32 nlist = 2;
if (this->nlist() != 0) {
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(2, this->nlist(), target);
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(2, this->nlist(), target);
}
// int32 index_file_size = 3;
......@@ -5528,13 +5528,6 @@ size_t Index::ByteSizeLong() const {
// Prevent compiler warnings about cached_has_bits being unused
(void) cached_has_bits;
// int64 nlist = 2;
if (this->nlist() != 0) {
total_size += 1 +
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int64Size(
this->nlist());
}
// int32 index_type = 1;
if (this->index_type() != 0) {
total_size += 1 +
......@@ -5542,6 +5535,13 @@ size_t Index::ByteSizeLong() const {
this->index_type());
}
// int32 nlist = 2;
if (this->nlist() != 0) {
total_size += 1 +
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size(
this->nlist());
}
// int32 index_file_size = 3;
if (this->index_file_size() != 0) {
total_size += 1 +
......@@ -5583,12 +5583,12 @@ void Index::MergeFrom(const Index& from) {
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
(void) cached_has_bits;
if (from.nlist() != 0) {
set_nlist(from.nlist());
}
if (from.index_type() != 0) {
set_index_type(from.index_type());
}
if (from.nlist() != 0) {
set_nlist(from.nlist());
}
if (from.index_file_size() != 0) {
set_index_file_size(from.index_file_size());
}
......@@ -5618,8 +5618,8 @@ bool Index::IsInitialized() const {
void Index::InternalSwap(Index* other) {
using std::swap;
_internal_metadata_.Swap(&other->_internal_metadata_);
swap(nlist_, other->nlist_);
swap(index_type_, other->index_type_);
swap(nlist_, other->nlist_);
swap(index_file_size_, other->index_file_size_);
swap(metric_type_, other->metric_type_);
}
......
......@@ -2316,21 +2316,21 @@ class Index :
// accessors -------------------------------------------------------
enum : int {
kNlistFieldNumber = 2,
kIndexTypeFieldNumber = 1,
kNlistFieldNumber = 2,
kIndexFileSizeFieldNumber = 3,
kMetricTypeFieldNumber = 4,
};
// int64 nlist = 2;
void clear_nlist();
::PROTOBUF_NAMESPACE_ID::int64 nlist() const;
void set_nlist(::PROTOBUF_NAMESPACE_ID::int64 value);
// int32 index_type = 1;
void clear_index_type();
::PROTOBUF_NAMESPACE_ID::int32 index_type() const;
void set_index_type(::PROTOBUF_NAMESPACE_ID::int32 value);
// int32 nlist = 2;
void clear_nlist();
::PROTOBUF_NAMESPACE_ID::int32 nlist() const;
void set_nlist(::PROTOBUF_NAMESPACE_ID::int32 value);
// int32 index_file_size = 3;
void clear_index_file_size();
::PROTOBUF_NAMESPACE_ID::int32 index_file_size() const;
......@@ -2346,8 +2346,8 @@ class Index :
class _Internal;
::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_;
::PROTOBUF_NAMESPACE_ID::int64 nlist_;
::PROTOBUF_NAMESPACE_ID::int32 index_type_;
::PROTOBUF_NAMESPACE_ID::int32 nlist_;
::PROTOBUF_NAMESPACE_ID::int32 index_file_size_;
::PROTOBUF_NAMESPACE_ID::int32 metric_type_;
mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_;
......@@ -3827,15 +3827,15 @@ inline void Index::set_index_type(::PROTOBUF_NAMESPACE_ID::int32 value) {
// @@protoc_insertion_point(field_set:milvus.grpc.Index.index_type)
}
// int64 nlist = 2;
// int32 nlist = 2;
inline void Index::clear_nlist() {
nlist_ = PROTOBUF_LONGLONG(0);
nlist_ = 0;
}
inline ::PROTOBUF_NAMESPACE_ID::int64 Index::nlist() const {
inline ::PROTOBUF_NAMESPACE_ID::int32 Index::nlist() const {
// @@protoc_insertion_point(field_get:milvus.grpc.Index.nlist)
return nlist_;
}
inline void Index::set_nlist(::PROTOBUF_NAMESPACE_ID::int64 value) {
inline void Index::set_nlist(::PROTOBUF_NAMESPACE_ID::int32 value) {
nlist_ = value;
// @@protoc_insertion_point(field_set:milvus.grpc.Index.nlist)
......
......@@ -61,7 +61,7 @@ static ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] =
const char descriptor_table_protodef_status_2eproto[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) =
"\n\014status.proto\022\013milvus.grpc\"D\n\006Status\022*\n"
"\nerror_code\030\001 \001(\0162\026.milvus.grpc.ErrorCod"
"e\022\016\n\006reason\030\002 \001(\t*\354\003\n\tErrorCode\022\013\n\007SUCCE"
"e\022\016\n\006reason\030\002 \001(\t*\230\004\n\tErrorCode\022\013\n\007SUCCE"
"SS\020\000\022\024\n\020UNEXPECTED_ERROR\020\001\022\022\n\016CONNECT_FA"
"ILED\020\002\022\025\n\021PERMISSION_DENIED\020\003\022\024\n\020TABLE_N"
"OT_EXISTS\020\004\022\024\n\020ILLEGAL_ARGUMENT\020\005\022\021\n\rILL"
......@@ -73,7 +73,9 @@ const char descriptor_table_protodef_status_2eproto[] PROTOBUF_SECTION_VARIABLE(
"TA_FAILED\020\017\022\020\n\014CACHE_FAILED\020\020\022\030\n\024CANNOT_"
"CREATE_FOLDER\020\021\022\026\n\022CANNOT_CREATE_FILE\020\022\022"
"\030\n\024CANNOT_DELETE_FOLDER\020\023\022\026\n\022CANNOT_DELE"
"TE_FILE\020\024\022\025\n\021BUILD_INDEX_ERROR\020\025b\006proto3"
"TE_FILE\020\024\022\025\n\021BUILD_INDEX_ERROR\020\025\022\021\n\rILLE"
"GAL_NLIST\020\026\022\027\n\023ILLEGAL_METRIC_TYPE\020\027b\006pr"
"oto3"
;
static const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable*const descriptor_table_status_2eproto_deps[1] = {
};
......@@ -83,7 +85,7 @@ static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_sta
static ::PROTOBUF_NAMESPACE_ID::internal::once_flag descriptor_table_status_2eproto_once;
static bool descriptor_table_status_2eproto_initialized = false;
const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable descriptor_table_status_2eproto = {
&descriptor_table_status_2eproto_initialized, descriptor_table_protodef_status_2eproto, "status.proto", 600,
&descriptor_table_status_2eproto_initialized, descriptor_table_protodef_status_2eproto, "status.proto", 644,
&descriptor_table_status_2eproto_once, descriptor_table_status_2eproto_sccs, descriptor_table_status_2eproto_deps, 1, 0,
schemas, file_default_instances, TableStruct_status_2eproto::offsets,
file_level_metadata_status_2eproto, 1, file_level_enum_descriptors_status_2eproto, file_level_service_descriptors_status_2eproto,
......@@ -121,6 +123,8 @@ bool ErrorCode_IsValid(int value) {
case 19:
case 20:
case 21:
case 22:
case 23:
return true;
default:
return false;
......
......@@ -91,12 +91,14 @@ enum ErrorCode : int {
CANNOT_DELETE_FOLDER = 19,
CANNOT_DELETE_FILE = 20,
BUILD_INDEX_ERROR = 21,
ILLEGAL_NLIST = 22,
ILLEGAL_METRIC_TYPE = 23,
ErrorCode_INT_MIN_SENTINEL_DO_NOT_USE_ = std::numeric_limits<::PROTOBUF_NAMESPACE_ID::int32>::min(),
ErrorCode_INT_MAX_SENTINEL_DO_NOT_USE_ = std::numeric_limits<::PROTOBUF_NAMESPACE_ID::int32>::max()
};
bool ErrorCode_IsValid(int value);
constexpr ErrorCode ErrorCode_MIN = SUCCESS;
constexpr ErrorCode ErrorCode_MAX = BUILD_INDEX_ERROR;
constexpr ErrorCode ErrorCode_MAX = ILLEGAL_METRIC_TYPE;
constexpr int ErrorCode_ARRAYSIZE = ErrorCode_MAX + 1;
const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* ErrorCode_descriptor();
......
......@@ -125,7 +125,7 @@ message Command {
*/
message Index {
int32 index_type = 1;
int64 nlist = 2;
int32 nlist = 2;
int32 index_file_size = 3;
int32 metric_type = 4;
}
......
......@@ -42,7 +42,6 @@ GrpcRequestHandler::HasTable(::grpc::ServerContext *context,
GrpcRequestHandler::DropTable(::grpc::ServerContext *context,
const ::milvus::grpc::TableName *request,
::milvus::grpc::Status *response) {
BaseTaskPtr task_ptr = DropTableTask::Create(request->table_name());
GrpcRequestScheduler::ExecTask(task_ptr, response);
return ::grpc::Status::OK;
......
......@@ -227,12 +227,32 @@ CreateIndexTask::OnExecute() {
return SetError(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists");
}
res = ValidationUtil::ValidateTableIndexType(index_param_.mutable_index()->index_type());
if(res != SERVER_SUCCESS) {
return SetError(res, "Invalid index type: " + std::to_string(index_param_.mutable_index()->index_type()));
}
res = ValidationUtil::ValidateTableIndexNlist(index_param_.mutable_index()->nlist());
if(res != SERVER_SUCCESS) {
return SetError(res, "Invalid index nlist: " + std::to_string(index_param_.mutable_index()->nlist()));
}
res = ValidationUtil::ValidateTableIndexMetricType(index_param_.mutable_index()->metric_type());
if(res != SERVER_SUCCESS) {
return SetError(res, "Invalid index metric type: " + std::to_string(index_param_.mutable_index()->metric_type()));
}
res = ValidationUtil::ValidateTableIndexFileSize(index_param_.mutable_index()->index_file_size());
if(res != SERVER_SUCCESS) {
return SetError(res, "Invalid index file size: " + std::to_string(index_param_.mutable_index()->index_file_size()));
}
//step 2: check table existence
engine::TableIndex index;
index.engine_type_ = index_param_.mutable_index()->index_type();
index.nlist = index_param_.mutable_index()->nlist();
index.index_file_size = index_param_.mutable_index()->index_file_size();
index.metric_type = index_param_.mutable_index()->metric_type();
index.nlist_ = index_param_.mutable_index()->nlist();
index.index_file_size_ = index_param_.mutable_index()->index_file_size();
index.metric_type_ = index_param_.mutable_index()->metric_type();
stat = DBWrapper::DB()->CreateIndex(table_name_, index);
if (!stat.ok()) {
return SetError(SERVER_BUILD_INDEX_ERROR, "Engine failed: " + stat.ToString());
......@@ -855,9 +875,9 @@ DescribeIndexTask::OnExecute() {
index_param_.mutable_table_name()->set_table_name(table_name_);
index_param_.mutable_index()->set_index_type(index.engine_type_);
index_param_.mutable_index()->set_nlist(index.nlist);
index_param_.mutable_index()->set_index_file_size(index.index_file_size);
index_param_.mutable_index()->set_metric_type(index.metric_type);
index_param_.mutable_index()->set_nlist(index.nlist_);
index_param_.mutable_index()->set_index_file_size(index.index_file_size_);
index_param_.mutable_index()->set_metric_type(index.metric_type_);
rc.ElapseFromBegin("totally cost");
} catch (std::exception &ex) {
......
......@@ -51,6 +51,9 @@ constexpr ServerError SERVER_ILLEGAL_SEARCH_RESULT = ToGlobalServerErrorCode(110
constexpr ServerError SERVER_CACHE_ERROR = ToGlobalServerErrorCode(111);
constexpr ServerError SERVER_WRITE_ERROR = ToGlobalServerErrorCode(112);
constexpr ServerError SERVER_INVALID_NPROBE = ToGlobalServerErrorCode(113);
constexpr ServerError SERVER_INVALID_INDEX_NLIST = ToGlobalServerErrorCode(114);
constexpr ServerError SERVER_INVALID_INDEX_METRIC_TYPE = ToGlobalServerErrorCode(115);
constexpr ServerError SERVER_INVALID_INDEX_FILE_SIZE = ToGlobalServerErrorCode(116);
constexpr ServerError SERVER_LICENSE_FILE_NOT_EXIST = ToGlobalServerErrorCode(500);
......
......@@ -10,6 +10,7 @@ namespace server {
constexpr size_t table_name_size_limit = 255;
constexpr int64_t table_dimension_limit = 16384;
constexpr int32_t index_file_size_limit = 4096; //index trigger size max = 4096 MB
ServerError
ValidationUtil::ValidateTableName(const std::string &table_name) {
......@@ -65,6 +66,32 @@ ValidationUtil::ValidateTableIndexType(int32_t index_type) {
return SERVER_SUCCESS;
}
ServerError
ValidationUtil::ValidateTableIndexNlist(int32_t nlist) {
if(nlist <= 0) {
return SERVER_INVALID_INDEX_NLIST;
}
return SERVER_SUCCESS;
}
ServerError
ValidationUtil::ValidateTableIndexFileSize(int32_t index_file_size) {
if(index_file_size <= 0 || index_file_size > index_file_size_limit) {
return SERVER_INVALID_INDEX_FILE_SIZE;
}
return SERVER_SUCCESS;
}
ServerError
ValidationUtil::ValidateTableIndexMetricType(int32_t metric_type) {
if(metric_type != (int32_t)engine::MetricType::L2 && metric_type != (int32_t)engine::MetricType::IP) {
return SERVER_INVALID_INDEX_METRIC_TYPE;
}
return SERVER_SUCCESS;
}
ServerError
ValidationUtil::ValidateGpuIndex(uint32_t gpu_index) {
int num_devices = 0;
......
......@@ -17,6 +17,15 @@ public:
static ServerError
ValidateTableIndexType(int32_t index_type);
static ServerError
ValidateTableIndexNlist(int32_t nlist);
static ServerError
ValidateTableIndexFileSize(int32_t index_file_size);
static ServerError
ValidateTableIndexMetricType(int32_t metric_type);
static ServerError
ValidateGpuIndex(uint32_t gpu_index);
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册