Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
3c706753
M
milvus
项目概览
milvus
/
milvus
11 个月 前同步成功
通知
260
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
前往新版Gitcode,体验更适合开发者的 AI 搜索 >>
提交
3c706753
编写于
6月 04, 2021
作者:
N
neza2017
提交者:
zhenshan.cao
6月 15, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
data node, save binlog to minIO, and let data service save these meta (#5618)
Signed-off-by:
N
yefu.chen
<
yefu.chen@zilliz.com
>
上级
91ef35ba
变更
2
隐藏空白更改
内联
并排
Showing
2 changed file
with
79 addition
and
18 deletion
+79
-18
internal/datanode/data_sync_service.go
internal/datanode/data_sync_service.go
+42
-1
internal/datanode/flow_graph_insert_buffer_node.go
internal/datanode/flow_graph_insert_buffer_node.go
+37
-17
未找到文件。
internal/datanode/data_sync_service.go
浏览文件 @
3c706753
...
...
@@ -13,10 +13,13 @@ package datanode
import
(
"context"
"fmt"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/msgstream"
"github.com/milvus-io/milvus/internal/proto/commonpb"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/types"
"github.com/milvus-io/milvus/internal/util/flowgraph"
"go.uber.org/zap"
...
...
@@ -30,6 +33,7 @@ type dataSyncService struct {
idAllocator
allocatorInterface
msFactory
msgstream
.
Factory
collectionID
UniqueID
dataService
types
.
DataService
}
func
newDataSyncService
(
ctx
context
.
Context
,
...
...
@@ -83,6 +87,43 @@ func (dsService *dataSyncService) initNodes(vchanPair *datapb.VchannelInfo) {
panic
(
err
)
}
saveBinlog
:=
func
(
fu
*
autoFlushUnit
)
error
{
id2path
:=
[]
*
datapb
.
ID2PathList
{}
checkPoints
:=
[]
*
datapb
.
CheckPoint
{}
for
k
,
v
:=
range
fu
.
field2Path
{
id2path
=
append
(
id2path
,
&
datapb
.
ID2PathList
{
ID
:
k
,
Paths
:
[]
string
{
v
}})
}
for
k
,
v
:=
range
fu
.
openSegCheckpoints
{
v
:=
v
checkPoints
=
append
(
checkPoints
,
&
datapb
.
CheckPoint
{
SegmentID
:
k
,
NumOfRows
:
fu
.
numRows
[
k
],
Position
:
&
v
,
})
}
req
:=
&
datapb
.
SaveBinlogPathsRequest
{
Base
:
&
commonpb
.
MsgBase
{
MsgType
:
0
,
//TOD msg type
MsgID
:
0
,
//TODO,msg id
Timestamp
:
0
,
//TODO, time stamp
SourceID
:
Params
.
NodeID
,
},
SegmentID
:
fu
.
segID
,
CollectionID
:
0
,
//TODO
Field2BinlogPaths
:
id2path
,
CheckPoints
:
checkPoints
,
Flushed
:
fu
.
flushed
,
}
rsp
,
err
:=
dsService
.
dataService
.
SaveBinlogPaths
(
dsService
.
ctx
,
req
)
if
err
!=
nil
{
return
fmt
.
Errorf
(
"data service save bin log path failed, err = %w"
,
err
)
}
if
rsp
.
ErrorCode
!=
commonpb
.
ErrorCode_Success
{
return
fmt
.
Errorf
(
"data service save bin log path failed, reason = %s"
,
rsp
.
Reason
)
}
return
nil
}
var
dmStreamNode
Node
=
newDmInputNode
(
dsService
.
ctx
,
dsService
.
msFactory
,
vchanPair
.
GetChannelName
(),
vchanPair
.
GetCheckPoints
())
var
ddNode
Node
=
newDDNode
()
var
insertBufferNode
Node
=
newInsertBufferNode
(
...
...
@@ -91,7 +132,7 @@ func (dsService *dataSyncService) initNodes(vchanPair *datapb.VchannelInfo) {
dsService
.
msFactory
,
dsService
.
idAllocator
,
dsService
.
flushChan
,
nil
,
//TODO,=================== call data service save binlog =========
saveBinlog
,
)
dsService
.
fg
.
AddNode
(
dmStreamNode
)
...
...
internal/datanode/flow_graph_insert_buffer_node.go
浏览文件 @
3c706753
...
...
@@ -64,10 +64,11 @@ type insertBufferNode struct {
}
type
autoFlushUnit
struct
{
collID
UniqueID
segID
UniqueID
numRows
int64
field2Path
map
[
UniqueID
]
string
openSegCheckpoints
map
[
UniqueID
]
internalpb
.
MsgPosition
numRows
map
[
UniqueID
]
int64
flushed
bool
}
...
...
@@ -503,15 +504,23 @@ func (ibNode *insertBufferNode) Operate(in []flowgraph.Msg) []flowgraph.Msg {
log
.
Debug
(
"segment is empty"
)
continue
}
segSta
,
err
:=
ibNode
.
replica
.
getSegmentStatisticsUpdates
(
fu
.
segID
)
if
err
!=
nil
{
log
.
Debug
(
"getSegmentStatisticsUpdates failed"
,
zap
.
Error
(
err
))
continue
}
fu
.
numRows
=
segSta
.
NumRows
fu
.
openSegCheckpoints
=
ibNode
.
replica
.
listOpenSegmentCheckPoint
()
fu
.
numRows
=
make
(
map
[
UniqueID
]
int64
)
for
k
:=
range
fu
.
openSegCheckpoints
{
segStat
,
err
:=
ibNode
.
replica
.
getSegmentStatisticsUpdates
(
k
)
if
err
!=
nil
{
log
.
Debug
(
"getSegmentStatisticsUpdates failed"
,
zap
.
Error
(
err
))
fu
.
numRows
=
nil
break
}
fu
.
numRows
[
k
]
=
segStat
.
NumRows
}
if
fu
.
numRows
==
nil
{
log
.
Debug
(
"failed on get segment num rows"
)
break
}
fu
.
flushed
=
false
if
ibNode
.
dsSaveBinlog
(
&
fu
)
!=
nil
{
if
err
:=
ibNode
.
dsSaveBinlog
(
&
fu
);
err
!=
nil
{
log
.
Debug
(
"data service save bin log path failed"
,
zap
.
Error
(
err
))
}
}
...
...
@@ -523,19 +532,30 @@ func (ibNode *insertBufferNode) Operate(in []flowgraph.Msg) []flowgraph.Msg {
currentSegID
:=
fmsg
.
segmentID
log
.
Debug
(
". Receiving flush message"
,
zap
.
Int64
(
"segmentID"
,
currentSegID
))
segSta
,
err
:=
ibNode
.
replica
.
getSegmentStatisticsUpdates
(
currentSegID
)
if
err
!=
nil
{
log
.
Debug
(
"getSegmentStatisticsUpdates failed"
,
zap
.
Error
(
err
))
checkPoints
:=
ibNode
.
replica
.
listOpenSegmentCheckPoint
()
numRows
:=
make
(
map
[
UniqueID
]
int64
)
for
k
:=
range
checkPoints
{
segStat
,
err
:=
ibNode
.
replica
.
getSegmentStatisticsUpdates
(
k
)
if
err
!=
nil
{
log
.
Debug
(
"getSegmentStatisticsUpdates failed"
,
zap
.
Error
(
err
))
numRows
=
nil
break
}
numRows
[
k
]
=
segStat
.
NumRows
}
if
numRows
==
nil
{
log
.
Debug
(
"failed on get segment num rows"
)
break
}
if
ibNode
.
insertBuffer
.
size
(
currentSegID
)
<=
0
{
log
.
Debug
(
".. Buffer empty ..."
)
ibNode
.
dsSaveBinlog
(
&
autoFlushUnit
{
collID
:
fmsg
.
collectionID
,
segID
:
currentSegID
,
numRows
:
segSta
.
N
umRows
,
field2Path
:
nil
,
openSegCheckpoints
:
ibNode
.
replica
.
listOpenSegmentCheckPoint
()
,
numRows
:
n
umRows
,
field2Path
:
map
[
UniqueID
]
string
{}
,
openSegCheckpoints
:
checkPoints
,
flushed
:
true
,
})
fmsg
.
dmlFlushedCh
<-
[]
*
datapb
.
ID2PathList
{{
ID
:
currentSegID
,
Paths
:
[]
string
{}}}
...
...
@@ -581,8 +601,8 @@ func (ibNode *insertBufferNode) Operate(in []flowgraph.Msg) []flowgraph.Msg {
fu
:=
<-
finishCh
close
(
finishCh
)
if
fu
.
field2Path
!=
nil
{
fu
.
numRows
=
segSta
.
N
umRows
fu
.
openSegCheckpoints
=
ibNode
.
replica
.
listOpenSegmentCheckPoint
()
fu
.
numRows
=
n
umRows
fu
.
openSegCheckpoints
=
checkPoints
fu
.
flushed
=
true
if
ibNode
.
dsSaveBinlog
(
&
fu
)
!=
nil
{
log
.
Debug
(
"data service save bin log path failed"
,
zap
.
Error
(
err
))
...
...
@@ -703,7 +723,7 @@ func flushSegment(collMeta *etcdpb.CollectionMeta, segID, partitionID, collID Un
}
replica
.
setSegmentCheckPoint
(
segID
)
flushUnit
<-
autoFlushUnit
{
segID
:
segID
,
field2Path
:
field2Path
}
flushUnit
<-
autoFlushUnit
{
collID
:
collID
,
segID
:
segID
,
field2Path
:
field2Path
}
clearFn
(
true
)
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录