Skip to content

M
milvus

milvus / milvus
大约 1 年 前同步成功

通知 261
Star 22476
Fork 2472
M
milvus

体验新版 GitCode，发现更多精彩内容 >>

提交 014c4fe8 编写于 作者: N neza2017 提交者： yefu.chen
浏览文件
操作

Add config of minSegmentSizeToEnableIndex 

Signed-off-by: Nneza2017 <yefu.chen@zilliz.com>
上级 ab21115d
展开全部 隐藏空白更改
内联 并排
Showing with 645 addition and 413 deletion
configs/advanced/master.yaml
浏览文件 @ 014c4fe8
......@@ -23,5 +23,6 @@ master:
IDAssignExpiration: 2000 # ms
maxPartitionNum: 4096
minSegmentSizeToEnableIndex: 4096
nodeID: 100
timeout: 3600 # time out, 5 seconds
\ No newline at end of file
internal/core/src/pb/etcd_meta.pb.cc
浏览文件 @ 014c4fe8
......@@ -295,6 +295,7 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_etcd_5fmeta_2eproto::offsets[]
PROTOBUF_FIELD_OFFSET(::milvus::proto::etcd::SegmentIndexInfo, fieldid_),
PROTOBUF_FIELD_OFFSET(::milvus::proto::etcd::SegmentIndexInfo, indexid_),
PROTOBUF_FIELD_OFFSET(::milvus::proto::etcd::SegmentIndexInfo, buildid_),
PROTOBUF_FIELD_OFFSET(::milvus::proto::etcd::SegmentIndexInfo, enable_index_),
~0u, // no _has_bits_
PROTOBUF_FIELD_OFFSET(::milvus::proto::etcd::CollectionMeta, _internal_metadata_),
~0u, // no _extensions_
......@@ -348,10 +349,10 @@ static const ::PROTOBUF_NAMESPACE_ID::internal::MigrationSchema schemas[] PROTOB
{ 33, -1, sizeof(::milvus::proto::etcd::FieldIndexInfo)},
{ 40, -1, sizeof(::milvus::proto::etcd::CollectionInfo)},
{ 50, -1, sizeof(::milvus::proto::etcd::SegmentIndexInfo)},
{ 59, -1, sizeof(::milvus::proto::etcd::CollectionMeta)},
{ 70, -1, sizeof(::milvus::proto::etcd::FieldBinlogFiles)},
{ 77, -1, sizeof(::milvus::proto::etcd::SegmentMeta)},
{ 92, -1, sizeof(::milvus::proto::etcd::FieldIndexMeta)},
{ 60, -1, sizeof(::milvus::proto::etcd::CollectionMeta)},
{ 71, -1, sizeof(::milvus::proto::etcd::FieldBinlogFiles)},
{ 78, -1, sizeof(::milvus::proto::etcd::SegmentMeta)},
{ 93, -1, sizeof(::milvus::proto::etcd::FieldIndexMeta)},
};
static ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] = {
......@@ -386,28 +387,29 @@ const char descriptor_table_protodef_etcd_5fmeta_2eproto[] PROTOBUF_SECTION_VARI
"vus.proto.schema.CollectionSchema\022\023\n\013cre"
"ate_time\030\003 \001(\004\022\024\n\014partitionIDs\030\004 \003(\003\0228\n\r"
"field_indexes\030\005 \003(\0132!.milvus.proto.etcd."
"FieldIndexInfo\"X\n\020SegmentIndexInfo\022\021\n\tse"
"FieldIndexInfo\"n\n\020SegmentIndexInfo\022\021\n\tse"
"gmentID\030\001 \001(\003\022\017\n\007fieldID\030\002 \001(\003\022\017\n\007indexI"
"D\030\003 \001(\003\022\017\n\007buildID\030\004 \001(\003\"\252\001\n\016CollectionM"
"eta\022\n\n\002ID\030\001 \001(\003\0225\n\006schema\030\002 \001(\0132%.milvus"
".proto.schema.CollectionSchema\022\023\n\013create"
"_time\030\003 \001(\004\022\022\n\nsegmentIDs\030\004 \003(\003\022\026\n\016parti"
"tion_tags\030\005 \003(\t\022\024\n\014partitionIDs\030\006 \003(\003\"9\n"
"\020FieldBinlogFiles\022\017\n\007fieldID\030\001 \001(\003\022\024\n\014bi"
"nlog_files\030\002 \003(\t\"\204\002\n\013SegmentMeta\022\021\n\tsegm"
"entID\030\001 \001(\003\022\024\n\014collectionID\030\002 \001(\003\022\025\n\rpar"
"tition_tag\030\003 \001(\t\022\025\n\rchannel_start\030\004 \001(\005\022"
"\023\n\013channel_end\030\005 \001(\005\022\021\n\topen_time\030\006 \001(\004\022"
"\022\n\nclose_time\030\007 \001(\004\022\020\n\010num_rows\030\010 \001(\003\022\020\n"
"\010mem_size\030\t \001(\003\022>\n\021binlog_file_paths\030\n \003"
"(\0132#.milvus.proto.etcd.FieldBinlogFiles\""
"\310\001\n\016FieldIndexMeta\022\021\n\tsegmentID\030\001 \001(\003\022\017\n"
"\007fieldID\030\002 \001(\003\022\017\n\007indexID\030\003 \001(\003\0227\n\014index"
"_params\030\004 \003(\0132!.milvus.proto.common.KeyV"
"aluePair\022.\n\005state\030\005 \001(\0162\037.milvus.proto.c"
"ommon.IndexState\022\030\n\020index_file_paths\030\006 \003"
"(\tB@Z>github.com/zilliztech/milvus-distr"
"ibuted/internal/proto/etcdpbb\006proto3"
"D\030\003 \001(\003\022\017\n\007buildID\030\004 \001(\003\022\024\n\014enable_index"
"\030\005 \001(\010\"\252\001\n\016CollectionMeta\022\n\n\002ID\030\001 \001(\003\0225\n"
"\006schema\030\002 \001(\0132%.milvus.proto.schema.Coll"
"ectionSchema\022\023\n\013create_time\030\003 \001(\004\022\022\n\nseg"
"mentIDs\030\004 \003(\003\022\026\n\016partition_tags\030\005 \003(\t\022\024\n"
"\014partitionIDs\030\006 \003(\003\"9\n\020FieldBinlogFiles\022"
"\017\n\007fieldID\030\001 \001(\003\022\024\n\014binlog_files\030\002 \003(\t\"\204"
"\002\n\013SegmentMeta\022\021\n\tsegmentID\030\001 \001(\003\022\024\n\014col"
"lectionID\030\002 \001(\003\022\025\n\rpartition_tag\030\003 \001(\t\022\025"
"\n\rchannel_start\030\004 \001(\005\022\023\n\013channel_end\030\005 \001"
"(\005\022\021\n\topen_time\030\006 \001(\004\022\022\n\nclose_time\030\007 \001("
"\004\022\020\n\010num_rows\030\010 \001(\003\022\020\n\010mem_size\030\t \001(\003\022>\n"
"\021binlog_file_paths\030\n \003(\0132#.milvus.proto."
"etcd.FieldBinlogFiles\"\310\001\n\016FieldIndexMeta"
"\022\021\n\tsegmentID\030\001 \001(\003\022\017\n\007fieldID\030\002 \001(\003\022\017\n\007"
"indexID\030\003 \001(\003\0227\n\014index_params\030\004 \003(\0132!.mi"
"lvus.proto.common.KeyValuePair\022.\n\005state\030"
"\005 \001(\0162\037.milvus.proto.common.IndexState\022\030"
"\n\020index_file_paths\030\006 \003(\tB@Z>github.com/z"
"illiztech/milvus-distributed/internal/pr"
"oto/etcdpbb\006proto3"
;
static const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable*const descriptor_table_etcd_5fmeta_2eproto_deps[2] = {
&::descriptor_table_common_2eproto,
......@@ -429,7 +431,7 @@ static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_etc
static ::PROTOBUF_NAMESPACE_ID::internal::once_flag descriptor_table_etcd_5fmeta_2eproto_once;
static bool descriptor_table_etcd_5fmeta_2eproto_initialized = false;
const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable descriptor_table_etcd_5fmeta_2eproto = {
&descriptor_table_etcd_5fmeta_2eproto_initialized, descriptor_table_protodef_etcd_5fmeta_2eproto, "etcd_meta.proto", 1556,
&descriptor_table_etcd_5fmeta_2eproto_initialized, descriptor_table_protodef_etcd_5fmeta_2eproto, "etcd_meta.proto", 1578,
&descriptor_table_etcd_5fmeta_2eproto_once, descriptor_table_etcd_5fmeta_2eproto_sccs, descriptor_table_etcd_5fmeta_2eproto_deps, 11, 2,
schemas, file_default_instances, TableStruct_etcd_5fmeta_2eproto::offsets,
file_level_metadata_etcd_5fmeta_2eproto, 11, file_level_enum_descriptors_etcd_5fmeta_2eproto, file_level_service_descriptors_etcd_5fmeta_2eproto,
......@@ -2791,15 +2793,15 @@ SegmentIndexInfo::SegmentIndexInfo(const SegmentIndexInfo& from)
_internal_metadata_(nullptr) {
_internal_metadata_.MergeFrom(from._internal_metadata_);
::memcpy(&segmentid_, &from.segmentid_,
static_cast<size_t>(reinterpret_cast<char*>(&buildid_) -
reinterpret_cast<char*>(&segmentid_)) + sizeof(buildid_));
static_cast<size_t>(reinterpret_cast<char*>(&enable_index_) -
reinterpret_cast<char*>(&segmentid_)) + sizeof(enable_index_));
// @@protoc_insertion_point(copy_constructor:milvus.proto.etcd.SegmentIndexInfo)
}
void SegmentIndexInfo::SharedCtor() {
::memset(&segmentid_, 0, static_cast<size_t>(
reinterpret_cast<char*>(&buildid_) -
reinterpret_cast<char*>(&segmentid_)) + sizeof(buildid_));
reinterpret_cast<char*>(&enable_index_) -
reinterpret_cast<char*>(&segmentid_)) + sizeof(enable_index_));
}
SegmentIndexInfo::~SegmentIndexInfo() {
......@@ -2826,8 +2828,8 @@ void SegmentIndexInfo::Clear() {
(void) cached_has_bits;
::memset(&segmentid_, 0, static_cast<size_t>(
reinterpret_cast<char*>(&buildid_) -
reinterpret_cast<char*>(&segmentid_)) + sizeof(buildid_));
reinterpret_cast<char*>(&enable_index_) -
reinterpret_cast<char*>(&segmentid_)) + sizeof(enable_index_));
_internal_metadata_.Clear();
}
......@@ -2867,6 +2869,13 @@ const char* SegmentIndexInfo::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPA
CHK_(ptr);
} else goto handle_unusual;
continue;
// bool enable_index = 5;
case 5:
if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 40)) {
enable_index_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr);
CHK_(ptr);
} else goto handle_unusual;
continue;
default: {
handle_unusual:
if ((tag & 7) == 4 || tag == 0) {
......@@ -2949,6 +2958,19 @@ bool SegmentIndexInfo::MergePartialFromCodedStream(
break;
}
// bool enable_index = 5;
case 5: {
if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (40 & 0xFF)) {
DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive<
bool, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_BOOL>(
input, &enable_index_)));
} else {
goto handle_unusual;
}
break;
}
default: {
handle_unusual:
if (tag == 0) {
......@@ -2996,6 +3018,11 @@ void SegmentIndexInfo::SerializeWithCachedSizes(
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64(4, this->buildid(), output);
}
// bool enable_index = 5;
if (this->enable_index() != 0) {
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBool(5, this->enable_index(), output);
}
if (_internal_metadata_.have_unknown_fields()) {
::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFields(
_internal_metadata_.unknown_fields(), output);
......@@ -3029,6 +3056,11 @@ void SegmentIndexInfo::SerializeWithCachedSizes(
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(4, this->buildid(), target);
}
// bool enable_index = 5;
if (this->enable_index() != 0) {
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(5, this->enable_index(), target);
}
if (_internal_metadata_.have_unknown_fields()) {
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFieldsToArray(
_internal_metadata_.unknown_fields(), target);
......@@ -3078,6 +3110,11 @@ size_t SegmentIndexInfo::ByteSizeLong() const {
this->buildid());
}
// bool enable_index = 5;
if (this->enable_index() != 0) {
total_size += 1 + 1;
}
int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size);
SetCachedSize(cached_size);
return total_size;
......@@ -3117,6 +3154,9 @@ void SegmentIndexInfo::MergeFrom(const SegmentIndexInfo& from) {
if (from.buildid() != 0) {
set_buildid(from.buildid());
}
if (from.enable_index() != 0) {
set_enable_index(from.enable_index());
}
}
void SegmentIndexInfo::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) {
......@@ -3144,6 +3184,7 @@ void SegmentIndexInfo::InternalSwap(SegmentIndexInfo* other) {
swap(fieldid_, other->fieldid_);
swap(indexid_, other->indexid_);
swap(buildid_, other->buildid_);
swap(enable_index_, other->enable_index_);
}
::PROTOBUF_NAMESPACE_ID::Metadata SegmentIndexInfo::GetMetadata() const {
......
internal/core/src/pb/etcd_meta.pb.h
浏览文件 @ 014c4fe8
......@@ -1189,6 +1189,7 @@ class SegmentIndexInfo :
kFieldIDFieldNumber = 2,
kIndexIDFieldNumber = 3,
kBuildIDFieldNumber = 4,
kEnableIndexFieldNumber = 5,
};
// int64 segmentID = 1;
void clear_segmentid();
......@@ -1210,6 +1211,11 @@ class SegmentIndexInfo :
::PROTOBUF_NAMESPACE_ID::int64 buildid() const;
void set_buildid(::PROTOBUF_NAMESPACE_ID::int64 value);
// bool enable_index = 5;
void clear_enable_index();
bool enable_index() const;
void set_enable_index(bool value);
// @@protoc_insertion_point(class_scope:milvus.proto.etcd.SegmentIndexInfo)
private:
class _Internal;
......@@ -1219,6 +1225,7 @@ class SegmentIndexInfo :
::PROTOBUF_NAMESPACE_ID::int64 fieldid_;
::PROTOBUF_NAMESPACE_ID::int64 indexid_;
::PROTOBUF_NAMESPACE_ID::int64 buildid_;
bool enable_index_;
mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_;
friend struct ::TableStruct_etcd_5fmeta_2eproto;
};
......@@ -2664,6 +2671,20 @@ inline void SegmentIndexInfo::set_buildid(::PROTOBUF_NAMESPACE_ID::int64 value)
// @@protoc_insertion_point(field_set:milvus.proto.etcd.SegmentIndexInfo.buildID)
}
// bool enable_index = 5;
inline void SegmentIndexInfo::clear_enable_index() {
enable_index_ = false;
}
inline bool SegmentIndexInfo::enable_index() const {
// @@protoc_insertion_point(field_get:milvus.proto.etcd.SegmentIndexInfo.enable_index)
return enable_index_;
}
inline void SegmentIndexInfo::set_enable_index(bool value) {
enable_index_ = value;
// @@protoc_insertion_point(field_set:milvus.proto.etcd.SegmentIndexInfo.enable_index)
}
// -------------------------------------------------------------------
// CollectionMeta
......
internal/core/src/pb/milvus.pb.cc
浏览文件 @ 014c4fe8
此差异已折叠。
internal/core/src/pb/milvus.pb.h
浏览文件 @ 014c4fe8
......@@ -4110,6 +4110,7 @@ class DescribeSegmentResponse :
kStatusFieldNumber = 1,
kIndexIDFieldNumber = 2,
kBuildIDFieldNumber = 3,
kEnableIndexFieldNumber = 4,
};
// .milvus.proto.common.Status status = 1;
bool has_status() const;
......@@ -4129,6 +4130,11 @@ class DescribeSegmentResponse :
::PROTOBUF_NAMESPACE_ID::int64 buildid() const;
void set_buildid(::PROTOBUF_NAMESPACE_ID::int64 value);
// bool enable_index = 4;
void clear_enable_index();
bool enable_index() const;
void set_enable_index(bool value);
// @@protoc_insertion_point(class_scope:milvus.proto.milvus.DescribeSegmentResponse)
private:
class _Internal;
......@@ -4137,6 +4143,7 @@ class DescribeSegmentResponse :
::milvus::proto::common::Status* status_;
::PROTOBUF_NAMESPACE_ID::int64 indexid_;
::PROTOBUF_NAMESPACE_ID::int64 buildid_;
bool enable_index_;
mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_;
friend struct ::TableStruct_milvus_2eproto;
};
......@@ -11561,6 +11568,20 @@ inline void DescribeSegmentResponse::set_buildid(::PROTOBUF_NAMESPACE_ID::int64
// @@protoc_insertion_point(field_set:milvus.proto.milvus.DescribeSegmentResponse.buildID)
}
// bool enable_index = 4;
inline void DescribeSegmentResponse::clear_enable_index() {
enable_index_ = false;
}
inline bool DescribeSegmentResponse::enable_index() const {
// @@protoc_insertion_point(field_get:milvus.proto.milvus.DescribeSegmentResponse.enable_index)
return enable_index_;
}
inline void DescribeSegmentResponse::set_enable_index(bool value) {
enable_index_ = value;
// @@protoc_insertion_point(field_set:milvus.proto.milvus.DescribeSegmentResponse.enable_index)
}
// -------------------------------------------------------------------
// ShowSegmentRequest
......
internal/distributed/masterservice/masterservice_test.go
浏览文件 @ 014c4fe8
......@@ -112,6 +112,9 @@ func TestGrpcService(t *testing.T) {
core.GetBinlogFilePathsFromDataServiceReq = func(segID typeutil.UniqueID, fieldID typeutil.UniqueID) ([]string, error) {
return []string{"file1", "file2", "file3"}, nil
}
core.GetNumRowsReq = func(segID typeutil.UniqueID) (int64, error) {
return cms.Params.MinSegmentSizeToEnableIndex, nil
}
var binlogLock sync.Mutex
binlogPathArray := make([]string, 0, 16)
......
internal/masterservice/master_service.go
浏览文件 @ 014c4fe8
......@@ -95,6 +95,7 @@ type Core struct {
//get binlog file path from data service,
GetBinlogFilePathsFromDataServiceReq func(segID typeutil.UniqueID, fieldID typeutil.UniqueID) ([]string, error)
GetNumRowsReq func(segID typeutil.UniqueID) (int64, error)
//call index builder's client to build index, return build id
BuildIndexReq func(binlog []string, typeParams []*commonpb.KeyValuePair, indexParams []*commonpb.KeyValuePair, indexID typeutil.UniqueID, indexName string) (typeutil.UniqueID, error)
......@@ -188,6 +189,9 @@ func (c *Core) checkInit() error {
if c.GetBinlogFilePathsFromDataServiceReq == nil {
return errors.New("GetBinlogFilePathsFromDataServiceReq is nil")
}
if c.GetNumRowsReq == nil {
return errors.New("GetNumRowsReq is nil")
}
if c.BuildIndexReq == nil {
return errors.New("BuildIndexReq is nil")
}
......@@ -629,7 +633,7 @@ func (c *Core) SetDataService(ctx context.Context, s types.DataService) error {
}
binlog, err := s.GetInsertBinlogPaths(ctx, &datapb.InsertBinlogPathRequest{
Base: &commonpb.MsgBase{
MsgType: 0, //TODO, msy type
MsgType: 0, //TODO, msg type
MsgID: 0,
Timestamp: ts,
SourceID: int64(Params.NodeID),
......@@ -649,6 +653,37 @@ func (c *Core) SetDataService(ctx context.Context, s types.DataService) error {
}
return nil, fmt.Errorf("binlog file not exist, segment id = %d, field id = %d", segID, fieldID)
}
c.GetNumRowsReq = func(segID typeutil.UniqueID) (int64, error) {
ts, err := c.tsoAllocator.Alloc(1)
if err != nil {
return 0, err
}
segInfo, err := s.GetSegmentInfo(ctx, &datapb.SegmentInfoRequest{
Base: &commonpb.MsgBase{
MsgType: 0, //TODO, msg type
MsgID: 0,
Timestamp: ts,
SourceID: int64(Params.NodeID),
},
SegmentIDs: []typeutil.UniqueID{segID},
})
if err != nil {
return 0, err
}
if segInfo.Status.ErrorCode != commonpb.ErrorCode_SUCCESS {
return 0, fmt.Errorf("GetSegmentInfo from data service failed, error = %s", segInfo.Status.Reason)
}
if len(segInfo.Infos) != 1 {
log.Debug("get segment info empty")
return 0, nil
}
if segInfo.Infos[0].FlushedTime == 0 {
log.Debug("segment id not flushed", zap.Int64("segment id", segID))
return 0, nil
}
return segInfo.Infos[0].NumRows, nil
}
return nil
}
......
internal/masterservice/master_service_test.go
浏览文件 @ 014c4fe8
......@@ -83,6 +83,21 @@ func (d *dataMock) GetInsertBinlogPaths(ctx context.Context, req *datapb.InsertB
return rst, nil
}
func (d *dataMock) GetSegmentInfo(ctx context.Context, req *datapb.SegmentInfoRequest) (*datapb.SegmentInfoResponse, error) {
return &datapb.SegmentInfoResponse{
Status: &commonpb.Status{
ErrorCode: commonpb.ErrorCode_SUCCESS,
Reason: "",
},
Infos: []*datapb.SegmentInfo{
{
FlushedTime: 100,
NumRows: Params.MinSegmentSizeToEnableIndex,
},
},
}, nil
}
type queryMock struct {
types.QueryService
collID []typeutil.UniqueID
......
internal/masterservice/param_table.go
浏览文件 @ 014c4fe8
......@@ -29,9 +29,10 @@ type ParamTable struct {
StatisticsChannel string
DataServiceSegmentChannel string // get from data service, data service create segment, or data node flush segment
MaxPartitionNum int64
DefaultPartitionName string
DefaultIndexName string
MaxPartitionNum int64
DefaultPartitionName string
DefaultIndexName string
MinSegmentSizeToEnableIndex int64
Timeout int
......@@ -60,6 +61,7 @@ func (p *ParamTable) Init() {
p.initStatisticsChannelName()
p.initMaxPartitionNum()
p.initMinSegmentSizeToEnableIndex()
p.initDefaultPartitionName()
p.initDefaultIndexName()
......@@ -149,6 +151,10 @@ func (p *ParamTable) initMaxPartitionNum() {
p.MaxPartitionNum = p.ParseInt64("master.maxPartitionNum")
}
func (p *ParamTable) initMinSegmentSizeToEnableIndex() {
p.MinSegmentSizeToEnableIndex = p.ParseInt64("master.minSegmentSizeToEnableIndex")
}
func (p *ParamTable) initDefaultPartitionName() {
name, err := p.Load("common.defaultPartitionName")
if err != nil {
......
internal/masterservice/param_table_test.go
浏览文件 @ 014c4fe8
......@@ -37,7 +37,10 @@ func TestParamTable(t *testing.T) {
t.Logf("master statistics channel = %s", Params.StatisticsChannel)
assert.NotEqual(t, Params.MaxPartitionNum, 0)
t.Logf("master initMaxPartitionNum = %d", Params.MaxPartitionNum)
t.Logf("master MaxPartitionNum = %d", Params.MaxPartitionNum)
assert.NotEqual(t, Params.MinSegmentSizeToEnableIndex, 0)
t.Logf("master MinSegmentSizeToEnableIndex = %d", Params.MinSegmentSizeToEnableIndex)
assert.NotEqual(t, Params.DefaultPartitionName, "")
t.Logf("default partition name = %s", Params.DefaultPartitionName)
......
internal/masterservice/task.go
浏览文件 @ 014c4fe8
......@@ -553,6 +553,7 @@ func (t *DescribeSegmentReqTask) Execute() error {
}
t.Rsp.IndexID = segIdxInfo.IndexID
t.Rsp.BuildID = segIdxInfo.BuildID
t.Rsp.EnableIndex = segIdxInfo.EnableIndex
return nil
}
......@@ -731,30 +732,41 @@ func (t *CreateIndexTask) BuildIndex() error {
if t.core.MetaTable.IsSegmentIndexed(t.segmentID, t.fieldSchema, t.indexParams) {
return nil
}
binlogs, err := t.core.GetBinlogFilePathsFromDataServiceReq(t.segmentID, t.fieldSchema.FieldID)
rows, err := t.core.GetNumRowsReq(t.segmentID)
if err != nil {
return err
}
var bldID typeutil.UniqueID
var bldID typeutil.UniqueID = 0
enableIdx := false
if rows < Params.MinSegmentSizeToEnableIndex {
log.Debug("num of is less than MinSegmentSizeToEnableIndex", zap.Int64("num rows", rows))
} else {
binlogs, err := t.core.GetBinlogFilePathsFromDataServiceReq(t.segmentID, t.fieldSchema.FieldID)
if err != nil {
return err
}
if len(t.indexParams) == 0 {
t.indexParams = make([]*commonpb.KeyValuePair, 0, len(t.fieldSchema.IndexParams))
for _, p := range t.fieldSchema.IndexParams {
t.indexParams = append(t.indexParams, &commonpb.KeyValuePair{
Key: p.Key,
Value: p.Value,
})
if len(t.indexParams) == 0 {
t.indexParams = make([]*commonpb.KeyValuePair, 0, len(t.fieldSchema.IndexParams))
for _, p := range t.fieldSchema.IndexParams {
t.indexParams = append(t.indexParams, &commonpb.KeyValuePair{
Key: p.Key,
Value: p.Value,
})
}
}
}
bldID, err = t.core.BuildIndexReq(binlogs, t.fieldSchema.TypeParams, t.indexParams, t.indexID, t.indexName)
if err != nil {
return err
bldID, err = t.core.BuildIndexReq(binlogs, t.fieldSchema.TypeParams, t.indexParams, t.indexID, t.indexName)
if err != nil {
return err
}
enableIdx = true
}
seg := etcdpb.SegmentIndexInfo{
SegmentID: t.segmentID,
FieldID: t.fieldSchema.FieldID,
IndexID: t.indexID,
BuildID: bldID,
SegmentID: t.segmentID,
FieldID: t.fieldSchema.FieldID,
IndexID: t.indexID,
BuildID: bldID,
EnableIndex: enableIdx,
}
err = t.core.MetaTable.AddIndex(&seg)
return err
......
internal/proto/etcd_meta.proto
浏览文件 @ 014c4fe8
......@@ -50,6 +50,7 @@ message SegmentIndexInfo {
int64 fieldID = 2;
int64 indexID = 3;
int64 buildID = 4;
bool enable_index = 5;
}
message CollectionMeta {
......
internal/proto/etcdpb/etcd_meta.pb.go
浏览文件 @ 014c4fe8
......@@ -373,6 +373,7 @@ type SegmentIndexInfo struct {
FieldID int64 `protobuf:"varint,2,opt,name=fieldID,proto3" json:"fieldID,omitempty"`
IndexID int64 `protobuf:"varint,3,opt,name=indexID,proto3" json:"indexID,omitempty"`
BuildID int64 `protobuf:"varint,4,opt,name=buildID,proto3" json:"buildID,omitempty"`
EnableIndex bool `protobuf:"varint,5,opt,name=enable_index,json=enableIndex,proto3" json:"enable_index,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
......@@ -431,6 +432,13 @@ func (m *SegmentIndexInfo) GetBuildID() int64 {
return 0
}
func (m *SegmentIndexInfo) GetEnableIndex() bool {
if m != nil {
return m.EnableIndex
}
return false
}
type CollectionMeta struct {
ID int64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"`
Schema *schemapb.CollectionSchema `protobuf:"bytes,2,opt,name=schema,proto3" json:"schema,omitempty"`
......@@ -764,58 +772,60 @@ func init() {
func init() { proto.RegisterFile("etcd_meta.proto", fileDescriptor_975d306d62b73e88) }
var fileDescriptor_975d306d62b73e88 = []byte{
// 845 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x54, 0x5f, 0x8f, 0xdb, 0x44,
0x10, 0x97, 0x93, 0x5c, 0x72, 0x9e, 0xfc, 0xb9, 0xbb, 0x7d, 0x32, 0xa5, 0xa5, 0xa9, 0xab, 0x42,
0x24, 0xc4, 0x45, 0x3a, 0x04, 0x6f, 0x20, 0x68, 0xc3, 0x49, 0x11, 0xa2, 0x0d, 0xce, 0x89, 0x07,
0x5e, 0xac, 0x8d, 0x3d, 0x97, 0xac, 0xe4, 0x5d, 0x07, 0xef, 0x9a, 0xde, 0xdd, 0x03, 0xe2, 0x15,
0x3e, 0x02, 0x5f, 0x11, 0xbe, 0x02, 0x12, 0xda, 0x3f, 0x71, 0xec, 0x6b, 0x8a, 0x50, 0x25, 0x1e,
0xe7, 0x37, 0x33, 0x3b, 0x33, 0xbf, 0xdf, 0xcc, 0xc2, 0x09, 0xaa, 0x24, 0x8d, 0x39, 0x2a, 0x7a,
0xbe, 0x2d, 0x72, 0x95, 0x93, 0x33, 0xce, 0xb2, 0x9f, 0x4b, 0x69, 0xad, 0x73, 0xed, 0x7d, 0x30,
0x48, 0x72, 0xce, 0x73, 0x61, 0xa1, 0x07, 0x03, 0x99, 0x6c, 0x90, 0xbb, 0xf0, 0xf0, 0x0f, 0x0f,
0xe0, 0x0a, 0x05, 0x15, 0xea, 0x3b, 0x54, 0x94, 0x8c, 0xa0, 0x35, 0x9f, 0x05, 0xde, 0xd8, 0x9b,
0xb4, 0xa3, 0xd6, 0x7c, 0x46, 0x3e, 0x84, 0x13, 0x51, 0xf2, 0xf8, 0xa7, 0x12, 0x8b, 0xdb, 0x58,
0xe4, 0x29, 0xca, 0xa0, 0x65, 0x9c, 0x43, 0x51, 0xf2, 0xef, 0x35, 0xfa, 0x52, 0x83, 0xe4, 0x63,
0x38, 0x63, 0x42, 0x62, 0xa1, 0xe2, 0x64, 0x43, 0x85, 0xc0, 0x6c, 0x3e, 0x93, 0x41, 0x7b, 0xdc,
0x9e, 0xf8, 0xd1, 0xa9, 0x75, 0xbc, 0xa8, 0x70, 0xf2, 0x11, 0x9c, 0xd8, 0x07, 0xab, 0xd8, 0xa0,
0x33, 0xf6, 0x26, 0x7e, 0x34, 0x32, 0x70, 0x15, 0x19, 0xfe, 0xea, 0x81, 0xbf, 0x28, 0xf2, 0x9b,
0xdb, 0x83, 0xbd, 0x7d, 0x0e, 0x3d, 0x9a, 0xa6, 0x05, 0x4a, 0xdb, 0x53, 0xff, 0xe2, 0xe1, 0x79,
0x63, 0x76, 0x37, 0xf5, 0xd7, 0x36, 0x26, 0xda, 0x05, 0xeb, 0x5e, 0x0b, 0x94, 0x65, 0x76, 0xa8,
0x57, 0xeb, 0xd8, 0xf7, 0x1a, 0xde, 0xc0, 0x70, 0x41, 0x0b, 0xc5, 0x14, 0xcb, 0xc5, 0x5c, 0x5c,
0xe7, 0xe4, 0x19, 0x8c, 0xb6, 0x3b, 0x20, 0x16, 0x94, 0xa3, 0xe9, 0xc8, 0x8f, 0x86, 0x15, 0xfa,
0x92, 0x72, 0x24, 0x63, 0xe8, 0x57, 0xc0, 0x7c, 0xe6, 0x48, 0xab, 0x43, 0xe4, 0x03, 0x00, 0x89,
0x6b, 0x8e, 0x42, 0xed, 0xea, 0xb7, 0xa3, 0x1a, 0x12, 0xfe, 0xee, 0x81, 0x3f, 0x17, 0x29, 0xde,
0x98, 0xb2, 0x8f, 0x00, 0x98, 0x36, 0xea, 0x25, 0x7d, 0x83, 0x98, 0x72, 0x01, 0xf4, 0x8c, 0x51,
0x95, 0xda, 0x99, 0x64, 0x06, 0x03, 0x9b, 0xb8, 0xa5, 0x05, 0xe5, 0xb6, 0x50, 0xff, 0xe2, 0xc9,
0x41, 0xaa, 0xbe, 0xc5, 0xdb, 0x1f, 0x68, 0x56, 0xe2, 0x82, 0xb2, 0x22, 0xea, 0x9b, 0xb4, 0x85,
0xc9, 0x0a, 0x67, 0x30, 0xba, 0x64, 0x98, 0xa5, 0xfb, 0x86, 0x02, 0xe8, 0x5d, 0xb3, 0x0c, 0xd3,
0x4a, 0x92, 0x9d, 0xf9, 0xf6, 0x5e, 0xc2, 0xbf, 0x3c, 0x18, 0xbd, 0xc8, 0xb3, 0x0c, 0x93, 0x8a,
0xce, 0xfb, 0xa2, 0x7e, 0x01, 0x5d, 0xbb, 0x9f, 0x4e, 0xd3, 0x67, 0xcd, 0x46, 0xdd, 0xee, 0xee,
0x1f, 0x59, 0x1a, 0x20, 0x72, 0x49, 0xe4, 0x31, 0xf4, 0x93, 0x02, 0xa9, 0xc2, 0x58, 0x31, 0x8e,
0x41, 0x7b, 0xec, 0x4d, 0x3a, 0x11, 0x58, 0xe8, 0x8a, 0x71, 0x24, 0x21, 0x0c, 0x6a, 0x22, 0xc8,
0xa0, 0x63, 0x78, 0x6f, 0x60, 0xe4, 0x12, 0x86, 0xd7, 0x7a, 0xd8, 0xd8, 0xf4, 0x8d, 0x32, 0x38,
0x3a, 0xc4, 0x99, 0x3e, 0xad, 0xf3, 0x26, 0x29, 0xd1, 0xe0, 0xba, 0xb2, 0x51, 0x86, 0xbf, 0xc0,
0xe9, 0xd2, 0xe9, 0x59, 0xd1, 0xf6, 0x10, 0xfc, 0x4a, 0x63, 0x37, 0xf6, 0x1e, 0xb0, 0xa4, 0xea,
0x17, 0x2a, 0xea, 0x9c, 0x59, 0x27, 0xb5, 0xdd, 0x14, 0x38, 0x80, 0xde, 0xaa, 0x64, 0x26, 0xa7,
0x63, 0x3d, 0xce, 0x0c, 0xff, 0x6c, 0xd0, 0x7d, 0xf0, 0x86, 0xfe, 0x6f, 0xba, 0x9b, 0x4b, 0xde,
0xb9, 0xbf, 0xe4, 0xcd, 0x6b, 0x52, 0x74, 0x6d, 0xb9, 0xae, 0x5f, 0xd3, 0x15, 0x5d, 0xcb, 0x37,
0x54, 0xeb, 0xbe, 0xa9, 0x5a, 0xf8, 0x0a, 0x4e, 0x8d, 0x1a, 0xcf, 0x99, 0xc8, 0xf2, 0xf5, 0x25,
0xcb, 0x50, 0xd6, 0xf9, 0xf4, 0x9a, 0x7c, 0x3e, 0x81, 0xc1, 0xca, 0x04, 0xc6, 0x7a, 0x6d, 0xf5,
0x0f, 0xa2, 0xcb, 0xf6, 0x57, 0xfb, 0xe4, 0xf0, 0xef, 0x16, 0xf4, 0x9d, 0x7e, 0x86, 0xbb, 0x7f,
0x97, 0x2e, 0x84, 0x41, 0xb2, 0x5f, 0xed, 0x9d, 0x7e, 0x0d, 0x8c, 0x3c, 0x85, 0x61, 0x63, 0x5a,
0x43, 0x98, 0x5f, 0x9b, 0xe3, 0x8a, 0xae, 0x75, 0x90, 0xfb, 0x97, 0x62, 0xa9, 0x68, 0xa1, 0x8c,
0xaa, 0x47, 0xd1, 0xc0, 0x81, 0x4b, 0x8d, 0x19, 0xe2, 0x5d, 0x10, 0x8a, 0x34, 0x38, 0x32, 0x21,
0xe0, 0xa0, 0x6f, 0x44, 0x4a, 0xde, 0x07, 0x3f, 0xdf, 0xa2, 0xb0, 0xba, 0x74, 0x8d, 0x2e, 0xc7,
0x1a, 0x30, 0xaa, 0x3c, 0x02, 0x48, 0xb2, 0x5c, 0x3a, 0xd5, 0x7a, 0xc6, 0xeb, 0x1b, 0xc4, 0xb8,
0xdf, 0x83, 0x63, 0xfd, 0xe9, 0x17, 0xf9, 0x6b, 0x19, 0x1c, 0x5b, 0xda, 0x44, 0xc9, 0xa3, 0xfc,
0xb5, 0xd4, 0x2e, 0x8e, 0x3c, 0x96, 0xec, 0x0e, 0x03, 0xdf, 0xba, 0x38, 0xf2, 0x25, 0xbb, 0x43,
0xf2, 0x0a, 0xce, 0x6a, 0x8c, 0xc6, 0x5b, 0xaa, 0x36, 0x32, 0x00, 0x73, 0x39, 0x4f, 0xdf, 0x76,
0x39, 0x35, 0xad, 0xa2, 0x93, 0x3d, 0xf7, 0x0b, 0x9d, 0x1b, 0xfe, 0xd6, 0xaa, 0x7f, 0x3a, 0xff,
0x41, 0x82, 0x77, 0xb9, 0x9e, 0xfb, 0xdf, 0x63, 0xe7, 0x5d, 0xbe, 0x47, 0xf2, 0x19, 0x1c, 0x49,
0x45, 0x15, 0x1a, 0x21, 0x46, 0x17, 0x8f, 0x0f, 0xa6, 0x9b, 0x31, 0x96, 0x3a, 0x2c, 0xb2, 0xd1,
0x64, 0x02, 0xa7, 0xb6, 0x78, 0x8d, 0xb1, 0xae, 0x59, 0xc4, 0x91, 0xc1, 0x2b, 0x2e, 0x9e, 0x7f,
0xf5, 0xe3, 0x97, 0x6b, 0xa6, 0x36, 0xe5, 0x4a, 0x3f, 0x36, 0xbd, 0x63, 0x59, 0xc6, 0xee, 0x14,
0x26, 0x9b, 0xa9, 0x2d, 0xf4, 0x49, 0xca, 0xa4, 0x2a, 0xd8, 0xaa, 0x54, 0x98, 0x4e, 0x99, 0x50,
0x58, 0x08, 0x9a, 0x4d, 0x4d, 0xf5, 0xa9, 0x66, 0x7b, 0xbb, 0x5a, 0x75, 0x8d, 0xf5, 0xe9, 0x3f,
0x01, 0x00, 0x00, 0xff, 0xff, 0x9d, 0x83, 0x5a, 0x23, 0x31, 0x08, 0x00, 0x00,
// 865 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x55, 0x5f, 0x6f, 0xe3, 0x44,
0x10, 0x97, 0xf3, 0xb7, 0x9e, 0x38, 0x69, 0xbb, 0x4f, 0xe6, 0xb8, 0xe3, 0x52, 0x9f, 0x0e, 0x22,
0x21, 0x1a, 0xa9, 0x08, 0xde, 0x40, 0x70, 0x17, 0x2a, 0x45, 0x88, 0xbb, 0xe0, 0x54, 0x3c, 0xf0,
0x62, 0x6d, 0xec, 0x69, 0xb2, 0x92, 0x77, 0x1d, 0xbc, 0x6b, 0xae, 0xed, 0x13, 0xaf, 0xf0, 0x11,
0x78, 0xe3, 0xf3, 0xc1, 0x57, 0x40, 0x42, 0xbb, 0xeb, 0x38, 0x76, 0x2f, 0x87, 0x50, 0x25, 0x1e,
0xe7, 0x37, 0x33, 0x9e, 0x99, 0xdf, 0x6f, 0x67, 0x0c, 0xc7, 0xa8, 0xe2, 0x24, 0xe2, 0xa8, 0xe8,
0xf9, 0x36, 0xcf, 0x54, 0x46, 0x4e, 0x39, 0x4b, 0x7f, 0x2e, 0xa4, 0xb5, 0xce, 0xb5, 0xf7, 0x91,
0x17, 0x67, 0x9c, 0x67, 0xc2, 0x42, 0x8f, 0x3c, 0x19, 0x6f, 0x90, 0x97, 0xe1, 0xc1, 0xef, 0x0e,
0xc0, 0x15, 0x0a, 0x2a, 0xd4, 0x77, 0xa8, 0x28, 0x19, 0x41, 0x6b, 0x3e, 0xf3, 0x9d, 0xb1, 0x33,
0x69, 0x87, 0xad, 0xf9, 0x8c, 0x7c, 0x08, 0xc7, 0xa2, 0xe0, 0xd1, 0x4f, 0x05, 0xe6, 0xb7, 0x91,
0xc8, 0x12, 0x94, 0x7e, 0xcb, 0x38, 0x87, 0xa2, 0xe0, 0xdf, 0x6b, 0xf4, 0x95, 0x06, 0xc9, 0xc7,
0x70, 0xca, 0x84, 0xc4, 0x5c, 0x45, 0xf1, 0x86, 0x0a, 0x81, 0xe9, 0x7c, 0x26, 0xfd, 0xf6, 0xb8,
0x3d, 0x71, 0xc3, 0x13, 0xeb, 0x78, 0x59, 0xe1, 0xe4, 0x23, 0x38, 0xb6, 0x1f, 0xac, 0x62, 0xfd,
0xce, 0xd8, 0x99, 0xb8, 0xe1, 0xc8, 0xc0, 0x55, 0x64, 0xf0, 0x8b, 0x03, 0xee, 0x22, 0xcf, 0x6e,
0x6e, 0x0f, 0xf6, 0xf6, 0x39, 0xf4, 0x69, 0x92, 0xe4, 0x28, 0x6d, 0x4f, 0x83, 0x8b, 0xc7, 0xe7,
0x8d, 0xd9, 0xcb, 0xa9, 0xbf, 0xb6, 0x31, 0xe1, 0x2e, 0x58, 0xf7, 0x9a, 0xa3, 0x2c, 0xd2, 0x43,
0xbd, 0x5a, 0xc7, 0xbe, 0xd7, 0xe0, 0x06, 0x86, 0x0b, 0x9a, 0x2b, 0xa6, 0x58, 0x26, 0xe6, 0xe2,
0x3a, 0x23, 0xcf, 0x61, 0xb4, 0xdd, 0x01, 0x91, 0xa0, 0x1c, 0x4d, 0x47, 0x6e, 0x38, 0xac, 0xd0,
0x57, 0x94, 0x23, 0x19, 0xc3, 0xa0, 0x02, 0xe6, 0xb3, 0x92, 0xb4, 0x3a, 0x44, 0x3e, 0x00, 0x90,
0xb8, 0xe6, 0x28, 0xd4, 0xae, 0x7e, 0x3b, 0xac, 0x21, 0xc1, 0x6f, 0x0e, 0xb8, 0x73, 0x91, 0xe0,
0x8d, 0x29, 0xfb, 0x04, 0x80, 0x69, 0xa3, 0x5e, 0xd2, 0x35, 0x88, 0x29, 0xe7, 0x43, 0xdf, 0x18,
0x55, 0xa9, 0x9d, 0x49, 0x66, 0xe0, 0xd9, 0xc4, 0x2d, 0xcd, 0x29, 0xb7, 0x85, 0x06, 0x17, 0x67,
0x07, 0xa9, 0xfa, 0x16, 0x6f, 0x7f, 0xa0, 0x69, 0x81, 0x0b, 0xca, 0xf2, 0x70, 0x60, 0xd2, 0x16,
0x26, 0x2b, 0x98, 0xc1, 0xe8, 0x92, 0x61, 0x9a, 0xec, 0x1b, 0xf2, 0xa1, 0x7f, 0xcd, 0x52, 0x4c,
0x2a, 0x49, 0x76, 0xe6, 0xbb, 0x7b, 0x09, 0xfe, 0x72, 0x60, 0xf4, 0x32, 0x4b, 0x53, 0x8c, 0x2b,
0x3a, 0xef, 0x8b, 0xfa, 0x05, 0xf4, 0xec, 0xfb, 0x2c, 0x35, 0x7d, 0xde, 0x6c, 0xb4, 0x7c, 0xbb,
0xfb, 0x8f, 0x2c, 0x0d, 0x10, 0x96, 0x49, 0xe4, 0x29, 0x0c, 0xe2, 0x1c, 0xa9, 0xc2, 0x48, 0x31,
0x8e, 0x7e, 0x7b, 0xec, 0x4c, 0x3a, 0x21, 0x58, 0xe8, 0x8a, 0x71, 0x24, 0x01, 0x78, 0x35, 0x11,
0xa4, 0xdf, 0x31, 0xbc, 0x37, 0x30, 0x72, 0x09, 0xc3, 0x6b, 0x3d, 0x6c, 0x64, 0xfa, 0x46, 0xe9,
0x77, 0x0f, 0x71, 0xa6, 0x57, 0xeb, 0xbc, 0x49, 0x4a, 0xe8, 0x5d, 0x57, 0x36, 0xca, 0xe0, 0x0f,
0x07, 0x4e, 0x96, 0xa5, 0xa0, 0x15, 0x6f, 0x8f, 0xc1, 0xad, 0x44, 0x2e, 0xe7, 0xde, 0x03, 0x96,
0x55, 0xfd, 0x89, 0x8a, 0xbb, 0xd2, 0xac, 0xb3, 0xda, 0x6e, 0x2a, 0xec, 0x43, 0x7f, 0x55, 0x30,
0x93, 0xd3, 0xb1, 0x9e, 0xd2, 0x24, 0x67, 0xe0, 0xa1, 0xa0, 0xab, 0x14, 0xed, 0x24, 0x7e, 0x77,
0xec, 0x4c, 0x8e, 0xc2, 0x81, 0xc5, 0x4c, 0x4b, 0xc1, 0x9f, 0x0d, 0x49, 0x0e, 0xee, 0xd9, 0xff,
0x2d, 0x49, 0x73, 0x11, 0x3a, 0xf7, 0x17, 0xa1, 0xb9, 0x71, 0x8a, 0xae, 0xad, 0x1e, 0xf5, 0x8d,
0xbb, 0xa2, 0x6b, 0xf9, 0x96, 0xb2, 0xbd, 0xb7, 0x95, 0x0d, 0x5e, 0xc3, 0x89, 0x51, 0xec, 0x05,
0x13, 0x69, 0xb6, 0xbe, 0x64, 0x29, 0xca, 0x3a, 0xe5, 0x4e, 0x93, 0xf2, 0x33, 0xf0, 0x56, 0x26,
0x30, 0xd2, 0x4f, 0x5b, 0x5f, 0x19, 0x5d, 0x76, 0xb0, 0xda, 0x27, 0x07, 0x7f, 0xb7, 0x60, 0x50,
0x4a, 0x6c, 0xb8, 0xfb, 0x77, 0x75, 0x03, 0xf0, 0xe2, 0xfd, 0xf3, 0xdf, 0x49, 0xdc, 0xc0, 0xc8,
0x33, 0x18, 0x36, 0xa6, 0x35, 0x84, 0xb9, 0xb5, 0x39, 0xae, 0xe8, 0x5a, 0x07, 0x95, 0xb7, 0x2b,
0x92, 0x8a, 0xe6, 0xca, 0x08, 0xdf, 0x0d, 0xbd, 0x12, 0x5c, 0x6a, 0xcc, 0x10, 0x5f, 0x06, 0xa1,
0x48, 0x8c, 0xf8, 0xdd, 0x10, 0x4a, 0xe8, 0x1b, 0x91, 0x90, 0xf7, 0xc1, 0xcd, 0xb6, 0x28, 0xac,
0x2e, 0x3d, 0xa3, 0xcb, 0x91, 0x06, 0x8c, 0x2a, 0x4f, 0x00, 0xe2, 0x34, 0x93, 0xa5, 0x6a, 0x7d,
0xe3, 0x75, 0x0d, 0x62, 0xdc, 0xef, 0xc1, 0x91, 0xfe, 0x31, 0xe4, 0xd9, 0x1b, 0xe9, 0x1f, 0x59,
0xda, 0x44, 0xc1, 0xc3, 0xec, 0x8d, 0xd4, 0x2e, 0x8e, 0x3c, 0x92, 0xec, 0x0e, 0x7d, 0xd7, 0xba,
0x38, 0xf2, 0x25, 0xbb, 0x43, 0xf2, 0x1a, 0x4e, 0x6b, 0x8c, 0x46, 0x5b, 0xaa, 0x36, 0xd2, 0x07,
0xb3, 0x5d, 0xcf, 0xde, 0xb5, 0x5d, 0x35, 0xad, 0xc2, 0xe3, 0x3d, 0xf7, 0x0b, 0x9d, 0x1b, 0xfc,
0xda, 0xaa, 0x1f, 0xa6, 0xff, 0x20, 0xc1, 0x43, 0x16, 0xec, 0xfe, 0x09, 0xed, 0x3c, 0xe4, 0x84,
0x92, 0xcf, 0xa0, 0x2b, 0x15, 0x55, 0x68, 0x84, 0x18, 0x5d, 0x3c, 0x3d, 0x98, 0x6e, 0xc6, 0x58,
0xea, 0xb0, 0xd0, 0x46, 0x93, 0x09, 0x9c, 0xd8, 0xe2, 0x35, 0xc6, 0x7a, 0xe6, 0x21, 0x8e, 0x0c,
0x5e, 0x71, 0xf1, 0xe2, 0xab, 0x1f, 0xbf, 0x5c, 0x33, 0xb5, 0x29, 0x56, 0xfa, 0x63, 0xd3, 0x3b,
0x96, 0xa6, 0xec, 0x4e, 0x61, 0xbc, 0x99, 0xda, 0x42, 0x9f, 0x24, 0x4c, 0xaa, 0x9c, 0xad, 0x0a,
0x85, 0xc9, 0x94, 0x09, 0x85, 0xb9, 0xa0, 0xe9, 0xd4, 0x54, 0x9f, 0x6a, 0xb6, 0xb7, 0xab, 0x55,
0xcf, 0x58, 0x9f, 0xfe, 0x13, 0x00, 0x00, 0xff, 0xff, 0xf3, 0x2b, 0x2b, 0x3e, 0x55, 0x08, 0x00,
0x00,
}
internal/proto/milvus.proto
浏览文件 @ 014c4fe8
......@@ -156,6 +156,7 @@ message DescribeSegmentResponse {
common.Status status = 1;
int64 indexID = 2;
int64 buildID = 3;
bool enable_index = 4;
}
message ShowSegmentRequest {
......
internal/proxynode/task.go
浏览文件 @ 014c4fe8
......@@ -1725,6 +1725,8 @@ func (gist *GetIndexStateTask) Execute(ctx context.Context) error {
getIndexStatesRequest := &indexpb.IndexStatesRequest{
IndexBuildIDs: make([]UniqueID, 0),
}
enableIndexBitMap := make([]bool, 0)
indexBuildIDs := make([]UniqueID, 0)
for _, segmentID := range allSegmentIDs {
describeSegmentRequest := &milvuspb.DescribeSegmentRequest{
......@@ -1742,12 +1744,17 @@ func (gist *GetIndexStateTask) Execute(ctx context.Context) error {
return err
}
if segmentDesc.IndexID == matchIndexID {
getIndexStatesRequest.IndexBuildIDs = append(getIndexStatesRequest.IndexBuildIDs, segmentDesc.BuildID)
indexBuildIDs = append(indexBuildIDs, segmentDesc.BuildID)
if segmentDesc.EnableIndex {
enableIndexBitMap = append(enableIndexBitMap, true)
} else {
enableIndexBitMap = append(enableIndexBitMap, false)
}
}
}
log.Println("GetIndexState:: len of allSegmentIDs:", len(allSegmentIDs), " len of IndexBuildIDs", len(getIndexStatesRequest.IndexBuildIDs))
if len(allSegmentIDs) != len(getIndexStatesRequest.IndexBuildIDs) {
log.Println("GetIndexState:: len of allSegmentIDs:", len(allSegmentIDs), " len of IndexBuildIDs", len(indexBuildIDs))
if len(allSegmentIDs) != len(indexBuildIDs) {
gist.result = &milvuspb.IndexStateResponse{
Status: &commonpb.Status{
ErrorCode: commonpb.ErrorCode_SUCCESS,
......@@ -1758,6 +1765,11 @@ func (gist *GetIndexStateTask) Execute(ctx context.Context) error {
return err
}
for idx, enableIndex := range enableIndexBitMap {
if enableIndex {
getIndexStatesRequest.IndexBuildIDs = append(getIndexStatesRequest.IndexBuildIDs, indexBuildIDs[idx])
}
}
states, err := gist.indexService.GetIndexStates(ctx, getIndexStatesRequest)
if err != nil {
return err
......
tests/python_test/collection/test_collection_count.py
浏览文件 @ 014c4fe8
......@@ -153,6 +153,7 @@ class TestCollectionCount:
# stats = connect.get_collection_stats(collection)
# assert stats[row_count] == insert_count * 2
@pytest.mark.tags("0331")
def test_collection_count_after_index_created(self, connect, collection, get_simple_index, insert_count):
'''
target: test count_entities, after index have been created
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或