Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
55f148f1
M
milvus
项目概览
milvus
/
milvus
12 个月 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
55f148f1
编写于
10月 15, 2021
作者:
Y
yukun
提交者:
GitHub
10月 15, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Add deleteMessage in flow_graph_filter_dm_node (#9504)
Signed-off-by:
N
fishpenguin
<
kun.yu@zilliz.com
>
上级
bb721f1c
变更
3
隐藏空白更改
内联
并排
Showing
3 changed file
with
157 addition
and
0 deletion
+157
-0
internal/querynode/flow_graph_filter_dm_node.go
internal/querynode/flow_graph_filter_dm_node.go
+67
-0
internal/querynode/flow_graph_filter_dm_node_test.go
internal/querynode/flow_graph_filter_dm_node_test.go
+89
-0
internal/querynode/flow_graph_message.go
internal/querynode/flow_graph_message.go
+1
-0
未找到文件。
internal/querynode/flow_graph_filter_dm_node.go
浏览文件 @
55f148f1
...
...
@@ -63,11 +63,13 @@ func (fdmNode *filterDmNode) Operate(in []flowgraph.Msg) []flowgraph.Msg {
var
iMsg
=
insertMsg
{
insertMessages
:
make
([]
*
msgstream
.
InsertMsg
,
0
),
deleteMessages
:
make
([]
*
msgstream
.
DeleteMsg
,
0
),
timeRange
:
TimeRange
{
timestampMin
:
msgStreamMsg
.
TimestampMin
(),
timestampMax
:
msgStreamMsg
.
TimestampMax
(),
},
}
for
_
,
msg
:=
range
msgStreamMsg
.
TsMessages
()
{
switch
msg
.
Type
()
{
case
commonpb
.
MsgType_Insert
:
...
...
@@ -75,6 +77,11 @@ func (fdmNode *filterDmNode) Operate(in []flowgraph.Msg) []flowgraph.Msg {
if
resMsg
!=
nil
{
iMsg
.
insertMessages
=
append
(
iMsg
.
insertMessages
,
resMsg
)
}
case
commonpb
.
MsgType_Delete
:
resMsg
:=
fdmNode
.
filterInvalidDeleteMessage
(
msg
.
(
*
msgstream
.
DeleteMsg
))
if
resMsg
!=
nil
{
iMsg
.
deleteMessages
=
append
(
iMsg
.
deleteMessages
,
resMsg
)
}
default
:
log
.
Warn
(
"Non supporting"
,
zap
.
Int32
(
"message type"
,
int32
(
msg
.
Type
())))
}
...
...
@@ -87,6 +94,66 @@ func (fdmNode *filterDmNode) Operate(in []flowgraph.Msg) []flowgraph.Msg {
return
[]
Msg
{
res
}
}
func
(
fdmNode
*
filterDmNode
)
filterInvalidDeleteMessage
(
msg
*
msgstream
.
DeleteMsg
)
*
msgstream
.
DeleteMsg
{
sp
,
ctx
:=
trace
.
StartSpanFromContext
(
msg
.
TraceCtx
())
msg
.
SetTraceCtx
(
ctx
)
defer
sp
.
Finish
()
// check if collection and partition exist
collection
:=
fdmNode
.
replica
.
hasCollection
(
msg
.
CollectionID
)
partition
:=
fdmNode
.
replica
.
hasPartition
(
msg
.
PartitionID
)
if
fdmNode
.
loadType
==
loadTypeCollection
&&
!
collection
{
log
.
Debug
(
"filter invalid delete message, collection dose not exist"
,
zap
.
Any
(
"collectionID"
,
msg
.
CollectionID
),
zap
.
Any
(
"partitionID"
,
msg
.
PartitionID
))
return
nil
}
if
fdmNode
.
loadType
==
loadTypePartition
&&
!
partition
{
log
.
Debug
(
"filter invalid delete message, partition dose not exist"
,
zap
.
Any
(
"collectionID"
,
msg
.
CollectionID
),
zap
.
Any
(
"partitionID"
,
msg
.
PartitionID
))
return
nil
}
if
msg
.
CollectionID
!=
fdmNode
.
collectionID
{
return
nil
}
// if the flow graph type is partition, check if the partition is target partition
if
fdmNode
.
loadType
==
loadTypePartition
&&
msg
.
PartitionID
!=
fdmNode
.
partitionID
{
log
.
Debug
(
"filter invalid delete message, partition is not the target partition"
,
zap
.
Any
(
"collectionID"
,
msg
.
CollectionID
),
zap
.
Any
(
"partitionID"
,
msg
.
PartitionID
))
return
nil
}
// check if partition has been released
if
fdmNode
.
loadType
==
loadTypeCollection
{
col
,
err
:=
fdmNode
.
replica
.
getCollectionByID
(
msg
.
CollectionID
)
if
err
!=
nil
{
log
.
Warn
(
err
.
Error
())
return
nil
}
if
err
=
col
.
checkReleasedPartitions
([]
UniqueID
{
msg
.
PartitionID
});
err
!=
nil
{
log
.
Warn
(
err
.
Error
())
return
nil
}
}
if
len
(
msg
.
PrimaryKeys
)
!=
len
(
msg
.
Timestamps
)
{
log
.
Warn
(
"Error, misaligned messages detected"
)
return
nil
}
if
len
(
msg
.
Timestamps
)
<=
0
{
log
.
Debug
(
"filter invalid delete message, no message"
,
zap
.
Any
(
"collectionID"
,
msg
.
CollectionID
),
zap
.
Any
(
"partitionID"
,
msg
.
PartitionID
))
return
nil
}
return
msg
}
func
(
fdmNode
*
filterDmNode
)
filterInvalidInsertMessage
(
msg
*
msgstream
.
InsertMsg
)
*
msgstream
.
InsertMsg
{
sp
,
ctx
:=
trace
.
StartSpanFromContext
(
msg
.
TraceCtx
())
msg
.
SetTraceCtx
(
ctx
)
...
...
internal/querynode/flow_graph_filter_dm_node_test.go
浏览文件 @
55f148f1
...
...
@@ -168,6 +168,95 @@ func TestFlowGraphFilterDmNode_filterInvalidInsertMessage(t *testing.T) {
})
}
func
TestFlowGraphFilterDmNode_filterInvalidDeleteMessage
(
t
*
testing
.
T
)
{
ctx
,
cancel
:=
context
.
WithCancel
(
context
.
Background
())
defer
cancel
()
t
.
Run
(
"delete valid test"
,
func
(
t
*
testing
.
T
)
{
msg
,
err
:=
genSimpleDeleteMsg
()
assert
.
NoError
(
t
,
err
)
fg
,
err
:=
getFilterDMNode
(
ctx
)
assert
.
NoError
(
t
,
err
)
res
:=
fg
.
filterInvalidDeleteMessage
(
msg
)
assert
.
NotNil
(
t
,
res
)
})
t
.
Run
(
"test delete no collection"
,
func
(
t
*
testing
.
T
)
{
msg
,
err
:=
genSimpleDeleteMsg
()
assert
.
NoError
(
t
,
err
)
msg
.
CollectionID
=
UniqueID
(
1000
)
fg
,
err
:=
getFilterDMNode
(
ctx
)
assert
.
NoError
(
t
,
err
)
res
:=
fg
.
filterInvalidDeleteMessage
(
msg
)
assert
.
Nil
(
t
,
res
)
})
t
.
Run
(
"test delete no partition"
,
func
(
t
*
testing
.
T
)
{
msg
,
err
:=
genSimpleDeleteMsg
()
assert
.
NoError
(
t
,
err
)
msg
.
PartitionID
=
UniqueID
(
1000
)
fg
,
err
:=
getFilterDMNode
(
ctx
)
assert
.
NoError
(
t
,
err
)
fg
.
loadType
=
loadTypePartition
res
:=
fg
.
filterInvalidDeleteMessage
(
msg
)
assert
.
Nil
(
t
,
res
)
})
t
.
Run
(
"test delete not target collection"
,
func
(
t
*
testing
.
T
)
{
msg
,
err
:=
genSimpleDeleteMsg
()
assert
.
NoError
(
t
,
err
)
fg
,
err
:=
getFilterDMNode
(
ctx
)
assert
.
NoError
(
t
,
err
)
fg
.
collectionID
=
UniqueID
(
1000
)
res
:=
fg
.
filterInvalidDeleteMessage
(
msg
)
assert
.
Nil
(
t
,
res
)
})
t
.
Run
(
"test delete not target partition"
,
func
(
t
*
testing
.
T
)
{
msg
,
err
:=
genSimpleDeleteMsg
()
assert
.
NoError
(
t
,
err
)
fg
,
err
:=
getFilterDMNode
(
ctx
)
assert
.
NoError
(
t
,
err
)
fg
.
loadType
=
loadTypePartition
fg
.
partitionID
=
UniqueID
(
1000
)
res
:=
fg
.
filterInvalidDeleteMessage
(
msg
)
assert
.
Nil
(
t
,
res
)
})
t
.
Run
(
"test delete released partition"
,
func
(
t
*
testing
.
T
)
{
msg
,
err
:=
genSimpleDeleteMsg
()
assert
.
NoError
(
t
,
err
)
fg
,
err
:=
getFilterDMNode
(
ctx
)
assert
.
NoError
(
t
,
err
)
col
,
err
:=
fg
.
replica
.
getCollectionByID
(
defaultCollectionID
)
assert
.
NoError
(
t
,
err
)
col
.
addReleasedPartition
(
defaultPartitionID
)
res
:=
fg
.
filterInvalidDeleteMessage
(
msg
)
assert
.
Nil
(
t
,
res
)
})
t
.
Run
(
"test delete misaligned messages"
,
func
(
t
*
testing
.
T
)
{
msg
,
err
:=
genSimpleDeleteMsg
()
assert
.
NoError
(
t
,
err
)
fg
,
err
:=
getFilterDMNode
(
ctx
)
assert
.
NoError
(
t
,
err
)
msg
.
Timestamps
=
make
([]
Timestamp
,
0
)
res
:=
fg
.
filterInvalidDeleteMessage
(
msg
)
assert
.
Nil
(
t
,
res
)
})
t
.
Run
(
"test delete no data"
,
func
(
t
*
testing
.
T
)
{
msg
,
err
:=
genSimpleDeleteMsg
()
assert
.
NoError
(
t
,
err
)
fg
,
err
:=
getFilterDMNode
(
ctx
)
assert
.
NoError
(
t
,
err
)
msg
.
Timestamps
=
make
([]
Timestamp
,
0
)
msg
.
PrimaryKeys
=
make
([]
IntPrimaryKey
,
0
)
res
:=
fg
.
filterInvalidDeleteMessage
(
msg
)
assert
.
Nil
(
t
,
res
)
})
}
func
TestFlowGraphFilterDmNode_Operate
(
t
*
testing
.
T
)
{
ctx
,
cancel
:=
context
.
WithCancel
(
context
.
Background
())
defer
cancel
()
...
...
internal/querynode/flow_graph_message.go
浏览文件 @
55f148f1
...
...
@@ -24,6 +24,7 @@ type MsgStreamMsg = flowgraph.MsgStreamMsg
type
insertMsg
struct
{
insertMessages
[]
*
msgstream
.
InsertMsg
deleteMessages
[]
*
msgstream
.
DeleteMsg
timeRange
TimeRange
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录