Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
6840486e
M
milvus
项目概览
milvus
/
milvus
12 个月 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
6840486e
编写于
6月 27, 2022
作者:
C
congqixia
提交者:
GitHub
6月 27, 2022
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Fix load progress complete before syncReplica (#17828)
Signed-off-by:
N
Congqi Xia
<
congqi.xia@zilliz.com
>
上级
f55ca5eb
变更
2
隐藏空白更改
内联
并排
Showing
2 changed file
with
45 addition
and
66 deletion
+45
-66
internal/querycoord/task.go
internal/querycoord/task.go
+32
-66
internal/querycoord/task_test.go
internal/querycoord/task_test.go
+13
-0
未找到文件。
internal/querycoord/task.go
浏览文件 @
6840486e
...
...
@@ -336,39 +336,8 @@ func (lct *loadCollectionTask) timestamp() Timestamp {
}
func
(
lct
*
loadCollectionTask
)
updateTaskProcess
()
{
collectionID
:=
lct
.
CollectionID
childTasks
:=
lct
.
getChildTask
()
allDone
:=
true
for
_
,
t
:=
range
childTasks
{
if
t
.
getState
()
!=
taskDone
{
allDone
=
false
break
}
// wait watchDeltaChannel task done after loading segment
nodeID
:=
getDstNodeIDByTask
(
t
)
if
t
.
msgType
()
==
commonpb
.
MsgType_LoadSegments
{
if
!
lct
.
cluster
.
HasWatchedDeltaChannel
(
lct
.
ctx
,
nodeID
,
collectionID
)
{
allDone
=
false
break
}
}
}
if
allDone
{
err
:=
lct
.
meta
.
setLoadPercentage
(
collectionID
,
0
,
100
,
querypb
.
LoadType_LoadCollection
)
if
err
!=
nil
{
log
.
Error
(
"loadCollectionTask: set load percentage to meta's collectionInfo"
,
zap
.
Int64
(
"collectionID"
,
collectionID
))
lct
.
setResultInfo
(
err
)
return
}
lct
.
once
.
Do
(
func
()
{
metrics
.
QueryCoordLoadCount
.
WithLabelValues
(
metrics
.
SuccessLabel
)
.
Inc
()
metrics
.
QueryCoordLoadLatency
.
WithLabelValues
()
.
Observe
(
float64
(
lct
.
elapseSpan
()
.
Milliseconds
()))
metrics
.
QueryCoordNumChildTasks
.
WithLabelValues
()
.
Sub
(
float64
(
len
(
lct
.
getChildTask
())))
})
}
//TODO move check all child task done to globalPostExecute
//this function shall just calculate intermediate progress
}
func
(
lct
*
loadCollectionTask
)
preExecute
(
ctx
context
.
Context
)
error
{
...
...
@@ -622,6 +591,18 @@ func (lct *loadCollectionTask) globalPostExecute(ctx context.Context) error {
}
}
err
=
lct
.
meta
.
setLoadPercentage
(
lct
.
CollectionID
,
0
,
100
,
querypb
.
LoadType_LoadCollection
)
if
err
!=
nil
{
log
.
Error
(
"loadCollectionTask: set load percentage to meta's collectionInfo"
,
zap
.
Int64
(
"collectionID"
,
lct
.
CollectionID
))
lct
.
setResultInfo
(
err
)
return
err
}
lct
.
once
.
Do
(
func
()
{
metrics
.
QueryCoordLoadCount
.
WithLabelValues
(
metrics
.
SuccessLabel
)
.
Inc
()
metrics
.
QueryCoordLoadLatency
.
WithLabelValues
()
.
Observe
(
float64
(
lct
.
elapseSpan
()
.
Milliseconds
()))
metrics
.
QueryCoordNumChildTasks
.
WithLabelValues
()
.
Sub
(
float64
(
len
(
lct
.
getChildTask
())))
})
return
nil
}
...
...
@@ -801,39 +782,8 @@ func (lpt *loadPartitionTask) timestamp() Timestamp {
}
func
(
lpt
*
loadPartitionTask
)
updateTaskProcess
()
{
collectionID
:=
lpt
.
CollectionID
partitionIDs
:=
lpt
.
PartitionIDs
childTasks
:=
lpt
.
getChildTask
()
allDone
:=
true
for
_
,
t
:=
range
childTasks
{
if
t
.
getState
()
!=
taskDone
{
allDone
=
false
}
// wait watchDeltaChannel task done after loading segment
nodeID
:=
getDstNodeIDByTask
(
t
)
if
t
.
msgType
()
==
commonpb
.
MsgType_LoadSegments
{
if
!
lpt
.
cluster
.
HasWatchedDeltaChannel
(
lpt
.
ctx
,
nodeID
,
collectionID
)
{
allDone
=
false
break
}
}
}
if
allDone
{
for
_
,
id
:=
range
partitionIDs
{
err
:=
lpt
.
meta
.
setLoadPercentage
(
collectionID
,
id
,
100
,
querypb
.
LoadType_LoadPartition
)
if
err
!=
nil
{
log
.
Error
(
"loadPartitionTask: set load percentage to meta's collectionInfo"
,
zap
.
Int64
(
"collectionID"
,
collectionID
),
zap
.
Int64
(
"partitionID"
,
id
))
lpt
.
setResultInfo
(
err
)
return
}
}
lpt
.
once
.
Do
(
func
()
{
metrics
.
QueryCoordLoadCount
.
WithLabelValues
(
metrics
.
SuccessLabel
)
.
Inc
()
metrics
.
QueryCoordLoadLatency
.
WithLabelValues
()
.
Observe
(
float64
(
lpt
.
elapseSpan
()
.
Milliseconds
()))
metrics
.
QueryCoordNumChildTasks
.
WithLabelValues
()
.
Sub
(
float64
(
len
(
lpt
.
getChildTask
())))
})
}
//TODO move check all child task done to globalPostExecute
//this function shall just calculate intermediate progress
}
func
(
lpt
*
loadPartitionTask
)
preExecute
(
context
.
Context
)
error
{
...
...
@@ -1057,6 +1007,7 @@ func (lpt *loadPartitionTask) postExecute(ctx context.Context) error {
func
(
lpt
*
loadPartitionTask
)
globalPostExecute
(
ctx
context
.
Context
)
error
{
collectionID
:=
lpt
.
CollectionID
partitionIDs
:=
lpt
.
PartitionIDs
collection
,
err
:=
lpt
.
meta
.
getCollectionInfoByID
(
collectionID
)
if
err
!=
nil
{
...
...
@@ -1080,6 +1031,21 @@ func (lpt *loadPartitionTask) globalPostExecute(ctx context.Context) error {
}
}
for
_
,
id
:=
range
partitionIDs
{
err
:=
lpt
.
meta
.
setLoadPercentage
(
collectionID
,
id
,
100
,
querypb
.
LoadType_LoadPartition
)
if
err
!=
nil
{
log
.
Error
(
"loadPartitionTask: set load percentage to meta's collectionInfo"
,
zap
.
Int64
(
"collectionID"
,
collectionID
),
zap
.
Int64
(
"partitionID"
,
id
))
lpt
.
setResultInfo
(
err
)
return
err
}
}
lpt
.
once
.
Do
(
func
()
{
metrics
.
QueryCoordLoadCount
.
WithLabelValues
(
metrics
.
SuccessLabel
)
.
Inc
()
metrics
.
QueryCoordLoadLatency
.
WithLabelValues
()
.
Observe
(
float64
(
lpt
.
elapseSpan
()
.
Milliseconds
()))
metrics
.
QueryCoordNumChildTasks
.
WithLabelValues
()
.
Sub
(
float64
(
len
(
lpt
.
getChildTask
())))
})
return
nil
}
...
...
internal/querycoord/task_test.go
浏览文件 @
6840486e
...
...
@@ -1347,6 +1347,12 @@ func TestUpdateTaskProcessWhenLoadSegment(t *testing.T) {
queryCoord
.
scheduler
.
processTask
(
watchDeltaChannel
)
collectionInfo
,
err
=
queryCoord
.
meta
.
getCollectionInfoByID
(
defaultCollectionID
)
assert
.
Nil
(
t
,
err
)
assert
.
Equal
(
t
,
int64
(
0
),
collectionInfo
.
InMemoryPercentage
)
err
=
loadCollectionTask
.
globalPostExecute
(
ctx
)
assert
.
NoError
(
t
,
err
)
collectionInfo
,
err
=
queryCoord
.
meta
.
getCollectionInfoByID
(
defaultCollectionID
)
assert
.
NoError
(
t
,
err
)
assert
.
Equal
(
t
,
int64
(
100
),
collectionInfo
.
InMemoryPercentage
)
err
=
removeAllSession
()
...
...
@@ -1365,6 +1371,7 @@ func TestUpdateTaskProcessWhenWatchDmChannel(t *testing.T) {
queryCoord
.
meta
.
addCollection
(
defaultCollectionID
,
querypb
.
LoadType_LoadCollection
,
genDefaultCollectionSchema
(
false
))
watchDmChannel
:=
genWatchDmChannelTask
(
ctx
,
queryCoord
,
node1
.
queryNodeID
)
loadCollectionTask
:=
watchDmChannel
.
getParentTask
()
collectionInfo
,
err
:=
queryCoord
.
meta
.
getCollectionInfoByID
(
defaultCollectionID
)
assert
.
Nil
(
t
,
err
)
...
...
@@ -1373,6 +1380,12 @@ func TestUpdateTaskProcessWhenWatchDmChannel(t *testing.T) {
collectionInfo
,
err
=
queryCoord
.
meta
.
getCollectionInfoByID
(
defaultCollectionID
)
assert
.
Nil
(
t
,
err
)
assert
.
Equal
(
t
,
int64
(
0
),
collectionInfo
.
InMemoryPercentage
)
err
=
loadCollectionTask
.
globalPostExecute
(
ctx
)
assert
.
NoError
(
t
,
err
)
collectionInfo
,
err
=
queryCoord
.
meta
.
getCollectionInfoByID
(
defaultCollectionID
)
assert
.
NoError
(
t
,
err
)
assert
.
Equal
(
t
,
int64
(
100
),
collectionInfo
.
InMemoryPercentage
)
err
=
removeAllSession
()
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录