Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
68bf983f
M
milvus
项目概览
milvus
/
milvus
11 个月 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
68bf983f
编写于
5月 31, 2021
作者:
C
congqixia
提交者:
zhenshan.cao
6月 15, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Add Collection ID in ddNode and filter logic (#5496)
Signed-off-by:
N
Congqi Xia
<
congqi.xia@zilliz.com
>
上级
1f3c9d5d
变更
3
隐藏空白更改
内联
并排
Showing
3 changed file
with
33 addition
and
14 deletion
+33
-14
internal/datanode/data_sync_service.go
internal/datanode/data_sync_service.go
+1
-1
internal/datanode/flow_graph_dd_node.go
internal/datanode/flow_graph_dd_node.go
+30
-11
internal/datanode/flow_graph_dd_node_test.go
internal/datanode/flow_graph_dd_node_test.go
+2
-2
未找到文件。
internal/datanode/data_sync_service.go
浏览文件 @
68bf983f
...
...
@@ -87,7 +87,7 @@ func (dsService *dataSyncService) initNodes(vchanPair *datapb.VchannelPair) {
var
ddStreamNode
Node
=
newDDInputNode
(
dsService
.
ctx
,
dsService
.
msFactory
,
vchanPair
.
GetDdlVchannelName
(),
vchanPair
.
GetDdlPosition
())
var
filterDmNode
Node
=
newFilteredDmNode
()
var
ddNode
Node
=
newDDNode
(
dsService
.
ctx
,
dsService
.
flushChan
,
dsService
.
replica
,
dsService
.
idAllocator
)
var
ddNode
Node
=
newDDNode
(
dsService
.
ctx
,
dsService
.
flushChan
,
dsService
.
replica
,
dsService
.
idAllocator
,
vchanPair
.
CollectionID
)
var
insertBufferNode
Node
=
newInsertBufferNode
(
dsService
.
ctx
,
dsService
.
replica
,
dsService
.
msFactory
,
dsService
.
idAllocator
)
var
gcNode
Node
=
newGCNode
(
dsService
.
replica
)
...
...
internal/datanode/flow_graph_dd_node.go
浏览文件 @
68bf983f
...
...
@@ -26,7 +26,6 @@ import (
miniokv
"github.com/milvus-io/milvus/internal/kv/minio"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/msgstream"
"github.com/milvus-io/milvus/internal/proto/commonpb"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/schemapb"
"github.com/milvus-io/milvus/internal/storage"
...
...
@@ -46,6 +45,8 @@ type ddNode struct {
kv
kv
.
BaseKV
replica
Replica
collectionID
UniqueID
}
type
ddData
struct
{
...
...
@@ -85,6 +86,10 @@ func (ddNode *ddNode) Operate(in []flowgraph.Msg) []flowgraph.Msg {
// TODO: add error handling
}
if
len
(
in
)
==
0
{
return
[]
flowgraph
.
Msg
{}
}
msMsg
,
ok
:=
in
[
0
]
.
(
*
MsgStreamMsg
)
if
!
ok
{
log
.
Error
(
"type assertion failed for MsgStreamMsg"
)
...
...
@@ -122,15 +127,27 @@ func (ddNode *ddNode) Operate(in []flowgraph.Msg) []flowgraph.Msg {
// do dd tasks
for
_
,
msg
:=
range
tsMessages
{
switch
msg
.
Type
()
{
case
commonpb
.
MsgType_CreateCollection
:
ddNode
.
createCollection
(
msg
.
(
*
msgstream
.
CreateCollectionMsg
))
case
commonpb
.
MsgType_DropCollection
:
ddNode
.
dropCollection
(
msg
.
(
*
msgstream
.
DropCollectionMsg
))
case
commonpb
.
MsgType_CreatePartition
:
ddNode
.
createPartition
(
msg
.
(
*
msgstream
.
CreatePartitionMsg
))
case
commonpb
.
MsgType_DropPartition
:
ddNode
.
dropPartition
(
msg
.
(
*
msgstream
.
DropPartitionMsg
))
switch
msg
:=
msg
.
(
type
)
{
case
*
msgstream
.
CreateCollectionMsg
:
if
msg
.
CollectionID
!=
ddNode
.
collectionID
{
continue
}
ddNode
.
createCollection
(
msg
)
case
*
msgstream
.
DropCollectionMsg
:
if
msg
.
CollectionID
!=
ddNode
.
collectionID
{
continue
}
ddNode
.
dropCollection
(
msg
)
case
*
msgstream
.
CreatePartitionMsg
:
if
msg
.
CollectionID
!=
ddNode
.
collectionID
{
continue
}
ddNode
.
createPartition
(
msg
)
case
*
msgstream
.
DropPartitionMsg
:
if
msg
.
CollectionID
!=
ddNode
.
collectionID
{
continue
}
ddNode
.
dropPartition
(
msg
)
default
:
log
.
Error
(
"Not supporting message type"
,
zap
.
Any
(
"Type"
,
msg
.
Type
()))
}
...
...
@@ -439,7 +456,7 @@ func (ddNode *ddNode) dropPartition(msg *msgstream.DropPartitionMsg) {
}
func
newDDNode
(
ctx
context
.
Context
,
inFlushCh
<-
chan
*
flushMsg
,
replica
Replica
,
idAllocator
allocatorInterface
)
*
ddNode
{
replica
Replica
,
idAllocator
allocatorInterface
,
collectionID
UniqueID
)
*
ddNode
{
maxQueueLength
:=
Params
.
FlowGraphMaxQueueLength
maxParallelism
:=
Params
.
FlowGraphMaxParallelism
...
...
@@ -478,5 +495,7 @@ func newDDNode(ctx context.Context, inFlushCh <-chan *flushMsg,
kv
:
minioKV
,
replica
:
replica
,
flushMap
:
&
sync
.
Map
{},
collectionID
:
collectionID
,
}
}
internal/datanode/flow_graph_dd_node_test.go
浏览文件 @
68bf983f
...
...
@@ -45,9 +45,9 @@ func TestFlowGraphDDNode_Operate(t *testing.T) {
defer
close
(
inFlushCh
)
replica
:=
newReplica
()
ddNode
:=
newDDNode
(
ctx
,
inFlushCh
,
replica
,
NewAllocatorFactory
())
collID
:=
UniqueID
(
0
)
ddNode
:=
newDDNode
(
ctx
,
inFlushCh
,
replica
,
NewAllocatorFactory
(),
collID
)
collName
:=
"col-test-0"
// create collection
createCollReq
:=
internalpb
.
CreateCollectionRequest
{
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录