Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
93416d55
M
milvus
项目概览
milvus
/
milvus
大约 1 年 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
93416d55
编写于
4月 29, 2022
作者:
Y
yah01
提交者:
GitHub
4月 29, 2022
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Fix LoadBalance loads the same segment multiple times (#16734)
Signed-off-by:
N
yah01
<
yang.cen@zilliz.com
>
上级
2f49db9d
变更
2
隐藏空白更改
内联
并排
Showing
2 changed file
with
27 addition
and
21 deletion
+27
-21
internal/querycoord/task.go
internal/querycoord/task.go
+19
-18
internal/querycoord/task_scheduler.go
internal/querycoord/task_scheduler.go
+8
-3
未找到文件。
internal/querycoord/task.go
浏览文件 @
93416d55
...
...
@@ -35,6 +35,7 @@ import (
"github.com/milvus-io/milvus/internal/proto/querypb"
"github.com/milvus-io/milvus/internal/util/funcutil"
"github.com/milvus-io/milvus/internal/util/timerecord"
"github.com/milvus-io/milvus/internal/util/typeutil"
)
const
timeoutForRPC
=
10
*
time
.
Second
...
...
@@ -1924,16 +1925,16 @@ func (lbt *loadBalanceTask) execute(ctx context.Context) error {
for
_
,
nodeID
:=
range
lbt
.
SourceNodeIDs
{
segmentID2Info
:=
make
(
map
[
UniqueID
]
*
querypb
.
SegmentInfo
)
dmChannel2WatchInfo
:=
make
(
map
[
string
]
*
querypb
.
DmChannelWatchInfo
)
recoveredCollectionIDs
:=
make
(
map
[
UniqueID
]
struct
{}
)
recoveredCollectionIDs
:=
make
(
typeutil
.
UniqueSet
)
segmentInfos
:=
lbt
.
meta
.
getSegmentInfosByNode
(
nodeID
)
for
_
,
segmentInfo
:=
range
segmentInfos
{
segmentID2Info
[
segmentInfo
.
SegmentID
]
=
segmentInfo
recoveredCollectionIDs
[
segmentInfo
.
CollectionID
]
=
struct
{}{}
recoveredCollectionIDs
.
Insert
(
segmentInfo
.
CollectionID
)
}
dmChannelWatchInfos
:=
lbt
.
meta
.
getDmChannelInfosByNodeID
(
nodeID
)
for
_
,
watchInfo
:=
range
dmChannelWatchInfos
{
dmChannel2WatchInfo
[
watchInfo
.
DmChannel
]
=
watchInfo
recoveredCollectionIDs
[
watchInfo
.
CollectionID
]
=
struct
{}{}
recoveredCollectionIDs
.
Insert
(
watchInfo
.
CollectionID
)
}
for
collectionID
:=
range
recoveredCollectionIDs
{
...
...
@@ -1977,27 +1978,27 @@ func (lbt *loadBalanceTask) execute(ctx context.Context) error {
for
_
,
segmentBingLog
:=
range
binlogs
{
segmentID
:=
segmentBingLog
.
SegmentID
if
info
,
ok
:=
segmentID2Info
[
segmentID
];
ok
{
if
_
,
ok
:=
segmentID2Info
[
segmentID
];
ok
{
segmentLoadInfo
:=
lbt
.
broker
.
generateSegmentLoadInfo
(
ctx
,
collectionID
,
partitionID
,
segmentBingLog
,
true
,
schema
)
msgBase
:=
proto
.
Clone
(
lbt
.
Base
)
.
(
*
commonpb
.
MsgBase
)
msgBase
.
MsgType
=
commonpb
.
MsgType_LoadSegments
for
_
,
replica
:=
range
info
.
ReplicaIds
{
loadSegmentReq
:=
&
querypb
.
LoadSegmentsRequest
{
Base
:
msgBase
,
Infos
:
[]
*
querypb
.
SegmentLoadInfo
{
segmentLoadInfo
},
Schema
:
schema
,
CollectionID
:
collectionID
,
LoadMeta
:
&
querypb
.
LoadMetaInfo
{
LoadType
:
collectionInfo
.
LoadType
,
CollectionID
:
collectionID
,
PartitionIDs
:
toRecoverPartitionIDs
,
},
ReplicaID
:
replica
,
}
loadSegmentReq
:=
&
querypb
.
LoadSegmentsRequest
{
Base
:
msgBase
,
Infos
:
[]
*
querypb
.
SegmentLoadInfo
{
segmentLoadInfo
},
Schema
:
schema
,
CollectionID
:
collectionID
,
loadSegmentReqs
=
append
(
loadSegmentReqs
,
loadSegmentReq
)
LoadMeta
:
&
querypb
.
LoadMetaInfo
{
LoadType
:
collectionInfo
.
LoadType
,
CollectionID
:
collectionID
,
PartitionIDs
:
toRecoverPartitionIDs
,
},
ReplicaID
:
replica
.
ReplicaID
,
}
loadSegmentReqs
=
append
(
loadSegmentReqs
,
loadSegmentReq
)
}
}
...
...
internal/querycoord/task_scheduler.go
浏览文件 @
93416d55
...
...
@@ -659,6 +659,8 @@ func (scheduler *TaskScheduler) scheduleLoop() {
// if triggerCondition == NodeDown, loadSegment and watchDmchannel request will keep reschedule until the success
// the node info has been deleted after assgining child task to triggerTask
// so it is necessary to update the meta of segment and dmchannel, or some data may be lost in meta
// triggerTask may be LoadCollection, LoadPartitions, LoadBalance
if
triggerTask
.
getResultInfo
()
.
ErrorCode
==
commonpb
.
ErrorCode_Success
||
triggerTask
.
getTriggerCondition
()
==
querypb
.
TriggerCondition_NodeDown
{
err
=
updateSegmentInfoFromTask
(
scheduler
.
ctx
,
triggerTask
,
scheduler
.
meta
)
if
err
!=
nil
{
...
...
@@ -919,8 +921,8 @@ func updateSegmentInfoFromTask(ctx context.Context, triggerTask task, meta Meta)
collectionID
:=
loadInfo
.
CollectionID
segmentID
:=
loadInfo
.
SegmentID
segment
,
ok
:=
segments
[
segmentID
]
if
!
ok
{
segment
,
saved
:=
segments
[
segmentID
]
if
!
saved
{
segment
=
&
querypb
.
SegmentInfo
{
SegmentID
:
segmentID
,
CollectionID
:
loadInfo
.
CollectionID
,
...
...
@@ -941,7 +943,10 @@ func updateSegmentInfoFromTask(ctx context.Context, triggerTask task, meta Meta)
if
_
,
ok
:=
segmentInfosToSave
[
collectionID
];
!
ok
{
segmentInfosToSave
[
collectionID
]
=
make
([]
*
querypb
.
SegmentInfo
,
0
)
}
segmentInfosToSave
[
collectionID
]
=
append
(
segmentInfosToSave
[
collectionID
],
segment
)
if
!
saved
{
segmentInfosToSave
[
collectionID
]
=
append
(
segmentInfosToSave
[
collectionID
],
segment
)
}
}
}
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录