Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
fe0a1bc2
M
milvus
项目概览
milvus
/
milvus
大约 1 年 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
fe0a1bc2
编写于
5月 15, 2023
作者:
Y
yihao.dai
提交者:
GitHub
5月 15, 2023
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Fix panic caused by wrong logic of getting unindexed segments (#24044)
Signed-off-by:
N
bigsheeper
<
yihao.dai@zilliz.com
>
上级
30415e1b
变更
2
显示空白变更内容
内联
并排
Showing
2 changed file
with
287 addition
and
13 deletion
+287
-13
internal/datacoord/handler.go
internal/datacoord/handler.go
+40
-13
internal/datacoord/server_test.go
internal/datacoord/server_test.go
+247
-0
未找到文件。
internal/datacoord/handler.go
浏览文件 @
fe0a1bc2
...
...
@@ -145,25 +145,52 @@ func (h *ServerHandler) GetQueryVChanPositions(channel *channel, partitionIDs ..
unIndexedIDs
.
Insert
(
s
.
GetID
())
}
}
hasUnIndexed
:=
true
for
hasUnIndexed
{
hasUnIndexed
=
false
// ================================================
// Segments blood relationship:
// a b
// \ /
// c d
// \ /
// e
//
// GC: a, b
// Indexed: c, d, e
// ||
// || (Index dropped and creating new index and not finished)
// \/
// UnIndexed: c, d, e
//
// Retrieve unIndexed expected result:
// unIndexed: c, d
// ================================================
isValid
:=
func
(
ids
...
UniqueID
)
bool
{
for
_
,
id
:=
range
ids
{
if
seg
,
ok
:=
segmentInfos
[
id
];
!
ok
||
seg
==
nil
{
return
false
}
}
return
true
}
retrieveUnIndexed
:=
func
()
bool
{
continueRetrieve
:=
false
for
id
:=
range
unIndexedIDs
{
// Indexed segments are compacted to a raw segment,
// replace it with the indexed ones
if
len
(
segmentInfos
[
id
]
.
GetCompactionFrom
())
>
0
{
unIndexedIDs
.
Remove
(
id
)
for
_
,
segID
:=
range
segmentInfos
[
id
]
.
GetCompactionFrom
()
{
if
indexed
.
Contain
(
segID
)
{
indexedIDs
.
Insert
(
segID
)
compactionFrom
:=
segmentInfos
[
id
]
.
GetCompactionFrom
()
if
len
(
compactionFrom
)
>
0
&&
isValid
(
compactionFrom
...
)
{
for
_
,
fromID
:=
range
compactionFrom
{
if
indexed
.
Contain
(
fromID
)
{
indexedIDs
.
Insert
(
fromID
)
}
else
{
unIndexedIDs
.
Insert
(
seg
ID
)
hasUnIndexed
=
true
unIndexedIDs
.
Insert
(
from
ID
)
continueRetrieve
=
true
}
}
droppedIDs
.
Remove
(
segmentInfos
[
id
]
.
GetCompactionFrom
()
...
)
unIndexedIDs
.
Remove
(
id
)
droppedIDs
.
Remove
(
compactionFrom
...
)
}
}
return
continueRetrieve
}
for
retrieveUnIndexed
()
{
}
return
&
datapb
.
VchannelInfo
{
...
...
internal/datacoord/server_test.go
浏览文件 @
fe0a1bc2
...
...
@@ -2371,6 +2371,253 @@ func TestGetQueryVChanPositions(t *testing.T) {
})
}
func
TestGetQueryVChanPositions_Retrieve_unIndexed
(
t
*
testing
.
T
)
{
t
.
Run
(
"ab GC-ed, cde unIndexed"
,
func
(
t
*
testing
.
T
)
{
svr
:=
newTestServer
(
t
,
nil
)
defer
closeTestServer
(
t
,
svr
)
schema
:=
newTestSchema
()
svr
.
meta
.
AddCollection
(
&
collectionInfo
{
ID
:
0
,
Schema
:
schema
,
})
err
:=
svr
.
meta
.
CreateIndex
(
&
model
.
Index
{
TenantID
:
""
,
CollectionID
:
0
,
FieldID
:
2
,
IndexID
:
1
,
})
assert
.
Nil
(
t
,
err
)
c
:=
&
datapb
.
SegmentInfo
{
ID
:
1
,
CollectionID
:
0
,
PartitionID
:
0
,
InsertChannel
:
"ch1"
,
State
:
commonpb
.
SegmentState_Dropped
,
DmlPosition
:
&
msgpb
.
MsgPosition
{
ChannelName
:
"ch1"
,
MsgID
:
[]
byte
{
1
,
2
,
3
},
MsgGroup
:
""
,
Timestamp
:
1
,
},
CompactionFrom
:
[]
int64
{
99
,
100
},
// a, b which have been GC-ed
}
err
=
svr
.
meta
.
AddSegment
(
NewSegmentInfo
(
c
))
assert
.
Nil
(
t
,
err
)
d
:=
&
datapb
.
SegmentInfo
{
ID
:
2
,
CollectionID
:
0
,
PartitionID
:
0
,
InsertChannel
:
"ch1"
,
State
:
commonpb
.
SegmentState_Dropped
,
DmlPosition
:
&
msgpb
.
MsgPosition
{
ChannelName
:
"ch1"
,
MsgID
:
[]
byte
{
1
,
2
,
3
},
MsgGroup
:
""
,
Timestamp
:
1
,
},
}
err
=
svr
.
meta
.
AddSegment
(
NewSegmentInfo
(
d
))
assert
.
Nil
(
t
,
err
)
e
:=
&
datapb
.
SegmentInfo
{
ID
:
3
,
CollectionID
:
0
,
PartitionID
:
1
,
InsertChannel
:
"ch1"
,
State
:
commonpb
.
SegmentState_Flushed
,
DmlPosition
:
&
msgpb
.
MsgPosition
{
ChannelName
:
"ch1"
,
MsgID
:
[]
byte
{
1
,
2
,
3
},
MsgGroup
:
""
,
Timestamp
:
1
,
},
CompactionFrom
:
[]
int64
{
1
,
2
},
// c, d
}
err
=
svr
.
meta
.
AddSegment
(
NewSegmentInfo
(
e
))
assert
.
Nil
(
t
,
err
)
vchan
:=
svr
.
handler
.
GetQueryVChanPositions
(
&
channel
{
Name
:
"ch1"
,
CollectionID
:
0
},
allPartitionID
)
assert
.
EqualValues
(
t
,
0
,
len
(
vchan
.
FlushedSegmentIds
))
assert
.
EqualValues
(
t
,
2
,
len
(
vchan
.
UnflushedSegmentIds
))
assert
.
ElementsMatch
(
t
,
[]
int64
{
c
.
GetID
(),
d
.
GetID
()},
vchan
.
UnflushedSegmentIds
)
// expected c, d
})
t
.
Run
(
"a GC-ed, bcde unIndexed"
,
func
(
t
*
testing
.
T
)
{
svr
:=
newTestServer
(
t
,
nil
)
defer
closeTestServer
(
t
,
svr
)
schema
:=
newTestSchema
()
svr
.
meta
.
AddCollection
(
&
collectionInfo
{
ID
:
0
,
Schema
:
schema
,
})
err
:=
svr
.
meta
.
CreateIndex
(
&
model
.
Index
{
TenantID
:
""
,
CollectionID
:
0
,
FieldID
:
2
,
IndexID
:
1
,
})
assert
.
Nil
(
t
,
err
)
a
:=
&
datapb
.
SegmentInfo
{
ID
:
99
,
CollectionID
:
0
,
PartitionID
:
0
,
InsertChannel
:
"ch1"
,
State
:
commonpb
.
SegmentState_Dropped
,
DmlPosition
:
&
msgpb
.
MsgPosition
{
ChannelName
:
"ch1"
,
MsgID
:
[]
byte
{
1
,
2
,
3
},
MsgGroup
:
""
,
Timestamp
:
1
,
},
}
err
=
svr
.
meta
.
AddSegment
(
NewSegmentInfo
(
a
))
assert
.
Nil
(
t
,
err
)
c
:=
&
datapb
.
SegmentInfo
{
ID
:
1
,
CollectionID
:
0
,
PartitionID
:
0
,
InsertChannel
:
"ch1"
,
State
:
commonpb
.
SegmentState_Dropped
,
DmlPosition
:
&
msgpb
.
MsgPosition
{
ChannelName
:
"ch1"
,
MsgID
:
[]
byte
{
1
,
2
,
3
},
MsgGroup
:
""
,
Timestamp
:
1
,
},
CompactionFrom
:
[]
int64
{
99
,
100
},
// a, b which have been GC-ed
}
err
=
svr
.
meta
.
AddSegment
(
NewSegmentInfo
(
c
))
assert
.
Nil
(
t
,
err
)
d
:=
&
datapb
.
SegmentInfo
{
ID
:
2
,
CollectionID
:
0
,
PartitionID
:
0
,
InsertChannel
:
"ch1"
,
State
:
commonpb
.
SegmentState_Dropped
,
DmlPosition
:
&
msgpb
.
MsgPosition
{
ChannelName
:
"ch1"
,
MsgID
:
[]
byte
{
1
,
2
,
3
},
MsgGroup
:
""
,
Timestamp
:
1
,
},
}
err
=
svr
.
meta
.
AddSegment
(
NewSegmentInfo
(
d
))
assert
.
Nil
(
t
,
err
)
e
:=
&
datapb
.
SegmentInfo
{
ID
:
3
,
CollectionID
:
0
,
PartitionID
:
1
,
InsertChannel
:
"ch1"
,
State
:
commonpb
.
SegmentState_Flushed
,
DmlPosition
:
&
msgpb
.
MsgPosition
{
ChannelName
:
"ch1"
,
MsgID
:
[]
byte
{
1
,
2
,
3
},
MsgGroup
:
""
,
Timestamp
:
1
,
},
CompactionFrom
:
[]
int64
{
1
,
2
},
// c, d
}
err
=
svr
.
meta
.
AddSegment
(
NewSegmentInfo
(
e
))
assert
.
Nil
(
t
,
err
)
vchan
:=
svr
.
handler
.
GetQueryVChanPositions
(
&
channel
{
Name
:
"ch1"
,
CollectionID
:
0
},
allPartitionID
)
assert
.
EqualValues
(
t
,
0
,
len
(
vchan
.
FlushedSegmentIds
))
assert
.
EqualValues
(
t
,
2
,
len
(
vchan
.
UnflushedSegmentIds
))
assert
.
ElementsMatch
(
t
,
[]
int64
{
c
.
GetID
(),
d
.
GetID
()},
vchan
.
UnflushedSegmentIds
)
// expected c, d
})
t
.
Run
(
"ab GC-ed, c unIndexed, de indexed"
,
func
(
t
*
testing
.
T
)
{
svr
:=
newTestServer
(
t
,
nil
)
defer
closeTestServer
(
t
,
svr
)
schema
:=
newTestSchema
()
svr
.
meta
.
AddCollection
(
&
collectionInfo
{
ID
:
0
,
Schema
:
schema
,
})
err
:=
svr
.
meta
.
CreateIndex
(
&
model
.
Index
{
TenantID
:
""
,
CollectionID
:
0
,
FieldID
:
2
,
IndexID
:
1
,
})
assert
.
Nil
(
t
,
err
)
c
:=
&
datapb
.
SegmentInfo
{
ID
:
1
,
CollectionID
:
0
,
PartitionID
:
0
,
InsertChannel
:
"ch1"
,
State
:
commonpb
.
SegmentState_Dropped
,
DmlPosition
:
&
msgpb
.
MsgPosition
{
ChannelName
:
"ch1"
,
MsgID
:
[]
byte
{
1
,
2
,
3
},
MsgGroup
:
""
,
Timestamp
:
1
,
},
CompactionFrom
:
[]
int64
{
99
,
100
},
// a, b which have been GC-ed
}
err
=
svr
.
meta
.
AddSegment
(
NewSegmentInfo
(
c
))
assert
.
Nil
(
t
,
err
)
d
:=
&
datapb
.
SegmentInfo
{
ID
:
2
,
CollectionID
:
0
,
PartitionID
:
0
,
InsertChannel
:
"ch1"
,
State
:
commonpb
.
SegmentState_Dropped
,
DmlPosition
:
&
msgpb
.
MsgPosition
{
ChannelName
:
"ch1"
,
MsgID
:
[]
byte
{
1
,
2
,
3
},
MsgGroup
:
""
,
Timestamp
:
1
,
},
}
err
=
svr
.
meta
.
AddSegment
(
NewSegmentInfo
(
d
))
assert
.
Nil
(
t
,
err
)
err
=
svr
.
meta
.
AddSegmentIndex
(
&
model
.
SegmentIndex
{
SegmentID
:
2
,
BuildID
:
1
,
IndexID
:
1
,
})
assert
.
Nil
(
t
,
err
)
err
=
svr
.
meta
.
FinishTask
(
&
indexpb
.
IndexTaskInfo
{
BuildID
:
1
,
State
:
commonpb
.
IndexState_Finished
,
})
assert
.
Nil
(
t
,
err
)
e
:=
&
datapb
.
SegmentInfo
{
ID
:
3
,
CollectionID
:
0
,
PartitionID
:
0
,
InsertChannel
:
"ch1"
,
State
:
commonpb
.
SegmentState_Flushed
,
DmlPosition
:
&
msgpb
.
MsgPosition
{
ChannelName
:
"ch1"
,
MsgID
:
[]
byte
{
1
,
2
,
3
},
MsgGroup
:
""
,
Timestamp
:
1
,
},
CompactionFrom
:
[]
int64
{
1
,
2
},
// c, d
}
err
=
svr
.
meta
.
AddSegment
(
NewSegmentInfo
(
e
))
assert
.
Nil
(
t
,
err
)
err
=
svr
.
meta
.
AddSegmentIndex
(
&
model
.
SegmentIndex
{
SegmentID
:
3
,
BuildID
:
2
,
IndexID
:
1
,
})
assert
.
Nil
(
t
,
err
)
err
=
svr
.
meta
.
FinishTask
(
&
indexpb
.
IndexTaskInfo
{
BuildID
:
2
,
State
:
commonpb
.
IndexState_Finished
,
})
assert
.
Nil
(
t
,
err
)
vchan
:=
svr
.
handler
.
GetQueryVChanPositions
(
&
channel
{
Name
:
"ch1"
,
CollectionID
:
0
},
allPartitionID
)
assert
.
EqualValues
(
t
,
1
,
len
(
vchan
.
FlushedSegmentIds
))
assert
.
EqualValues
(
t
,
0
,
len
(
vchan
.
UnflushedSegmentIds
))
assert
.
ElementsMatch
(
t
,
[]
int64
{
e
.
GetID
()},
vchan
.
FlushedSegmentIds
)
// expected e
})
}
func
TestShouldDropChannel
(
t
*
testing
.
T
)
{
type
myRootCoord
struct
{
mocks
.
RootCoord
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录