Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
af1d84e5
M
milvus
项目概览
milvus
/
milvus
11 个月 前同步成功
通知
260
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
前往新版Gitcode,体验更适合开发者的 AI 搜索 >>
未验证
提交
af1d84e5
编写于
6月 20, 2023
作者:
X
Xiaofan
提交者:
GitHub
6月 20, 2023
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Support sync when cp is lagged (#24989)
Signed-off-by:
N
xiaofan-luan
<
xiaofan.luan@zilliz.com
>
上级
a6310050
变更
6
隐藏空白更改
内联
并排
Showing
6 changed file
with
139 addition
and
15 deletion
+139
-15
internal/core/conanfile.py
internal/core/conanfile.py
+1
-1
internal/datanode/channel_meta.go
internal/datanode/channel_meta.go
+1
-0
internal/datanode/data_sync_service_test.go
internal/datanode/data_sync_service_test.go
+16
-12
internal/datanode/segment_sync_policy.go
internal/datanode/segment_sync_policy.go
+43
-2
internal/datanode/segment_sync_policy_test.go
internal/datanode/segment_sync_policy_test.go
+68
-0
pkg/util/paramtable/component_param.go
pkg/util/paramtable/component_param.go
+10
-0
未找到文件。
internal/core/conanfile.py
浏览文件 @
af1d84e5
...
...
@@ -8,7 +8,7 @@ class MilvusConan(ConanFile):
"boost/1.79.0"
,
"onetbb/2021.7.0"
,
"nlohmann_json/3.11.2"
,
"zstd/1.5.
5
"
,
"zstd/1.5.
4
"
,
"lz4/1.9.4"
,
"snappy/1.1.9"
,
"lzo/2.10"
,
...
...
internal/datanode/channel_meta.go
浏览文件 @
af1d84e5
...
...
@@ -156,6 +156,7 @@ func newChannel(channelName string, collID UniqueID, schema *schemapb.Collection
syncPolicies
:
[]
segmentSyncPolicy
{
syncPeriodically
(),
syncMemoryTooHigh
(),
syncCPLagTooBehind
(),
},
metaService
:
metaService
,
...
...
internal/datanode/data_sync_service_test.go
浏览文件 @
af1d84e5
...
...
@@ -435,14 +435,18 @@ func TestDataSyncService_Close(t *testing.T) {
paramtable
.
Get
()
.
Reset
(
Params
.
DataNodeCfg
.
FlushInsertBufferSize
.
Key
)
channel
:=
newChannel
(
insertChannelName
,
collMeta
.
ID
,
collMeta
.
GetSchema
(),
mockRootCoord
,
cm
)
channel
.
syncPolicies
=
[]
segmentSyncPolicy
{
syncPeriodically
(),
syncMemoryTooHigh
(),
}
atimeTickSender
:=
newTimeTickManager
(
mockDataCoord
,
0
)
sync
,
err
:=
newDataSyncService
(
ctx
,
flushChan
,
resendTTChan
,
channel
,
alloc
,
dispClient
,
factory
,
vchan
,
signalCh
,
mockDataCoord
,
newCache
(),
cm
,
newCompactionExecutor
(),
genTestTickler
(),
0
,
atimeTickSender
)
sync
Service
,
err
:=
newDataSyncService
(
ctx
,
flushChan
,
resendTTChan
,
channel
,
alloc
,
dispClient
,
factory
,
vchan
,
signalCh
,
mockDataCoord
,
newCache
(),
cm
,
newCompactionExecutor
(),
genTestTickler
(),
0
,
atimeTickSender
)
assert
.
NoError
(
t
,
err
)
sync
.
flushListener
=
make
(
chan
*
segmentFlushPack
,
10
)
defer
close
(
sync
.
flushListener
)
sync
Service
.
flushListener
=
make
(
chan
*
segmentFlushPack
,
10
)
defer
close
(
sync
Service
.
flushListener
)
sync
.
start
()
sync
Service
.
start
()
var
(
dataFactory
=
NewDataFactory
()
...
...
@@ -525,26 +529,26 @@ func TestDataSyncService_Close(t *testing.T) {
assert
.
NoError
(
t
,
err
)
// wait for delete, no auto flush leads to all data in buffer.
require
.
Eventually
(
t
,
func
()
bool
{
return
sync
.
delBufferManager
.
GetEntriesNum
(
1
)
==
1
},
require
.
Eventually
(
t
,
func
()
bool
{
return
sync
Service
.
delBufferManager
.
GetEntriesNum
(
1
)
==
1
},
5
*
time
.
Second
,
100
*
time
.
Millisecond
)
assert
.
Equal
(
t
,
0
,
len
(
sync
.
flushListener
))
assert
.
Equal
(
t
,
0
,
len
(
sync
Service
.
flushListener
))
// close will trigger a force sync
sync
.
close
()
assert
.
Eventually
(
t
,
func
()
bool
{
return
len
(
sync
.
flushListener
)
==
1
},
sync
Service
.
close
()
assert
.
Eventually
(
t
,
func
()
bool
{
return
len
(
sync
Service
.
flushListener
)
==
1
},
5
*
time
.
Second
,
100
*
time
.
Millisecond
)
flushPack
,
ok
:=
<-
sync
.
flushListener
flushPack
,
ok
:=
<-
sync
Service
.
flushListener
assert
.
True
(
t
,
ok
)
assert
.
Equal
(
t
,
UniqueID
(
1
),
flushPack
.
segmentID
)
assert
.
True
(
t
,
len
(
flushPack
.
insertLogs
)
==
12
)
assert
.
True
(
t
,
len
(
flushPack
.
statsLogs
)
==
1
)
assert
.
True
(
t
,
len
(
flushPack
.
deltaLogs
)
==
1
)
<-
sync
.
ctx
.
Done
()
<-
sync
Service
.
ctx
.
Done
()
// Double close is safe
sync
.
close
()
<-
sync
.
ctx
.
Done
()
sync
Service
.
close
()
<-
sync
Service
.
ctx
.
Done
()
}
func
genBytes
()
(
rawData
[]
byte
)
{
...
...
internal/datanode/segment_sync_policy.go
浏览文件 @
af1d84e5
...
...
@@ -46,8 +46,7 @@ func syncPeriodically() segmentSyncPolicy {
}
}
if
len
(
segsToSync
)
>
0
{
log
.
Debug
(
"sync segment periodically"
,
zap
.
Int64s
(
"segmentID"
,
segsToSync
))
log
.
Info
(
"sync segment periodically"
,
zap
.
Int64s
(
"segmentID"
,
segsToSync
))
}
return
segsToSync
}
...
...
@@ -76,3 +75,45 @@ func syncMemoryTooHigh() segmentSyncPolicy {
return
syncSegments
}
}
// syncCPLagTooBehind force sync the segments lagging too behind the channel checkPoint
func
syncCPLagTooBehind
()
segmentSyncPolicy
{
segmentMinTs
:=
func
(
segment
*
Segment
)
uint64
{
var
minTs
uint64
=
math
.
MaxUint64
if
segment
.
curInsertBuf
!=
nil
&&
segment
.
curInsertBuf
.
startPos
!=
nil
&&
segment
.
curInsertBuf
.
startPos
.
Timestamp
<
minTs
{
minTs
=
segment
.
curInsertBuf
.
startPos
.
Timestamp
}
if
segment
.
curDeleteBuf
!=
nil
&&
segment
.
curDeleteBuf
.
startPos
!=
nil
&&
segment
.
curDeleteBuf
.
startPos
.
Timestamp
<
minTs
{
minTs
=
segment
.
curDeleteBuf
.
startPos
.
Timestamp
}
for
_
,
ib
:=
range
segment
.
historyInsertBuf
{
if
ib
!=
nil
&&
ib
.
startPos
!=
nil
&&
ib
.
startPos
.
Timestamp
<
minTs
{
minTs
=
ib
.
startPos
.
Timestamp
}
}
for
_
,
db
:=
range
segment
.
historyDeleteBuf
{
if
db
!=
nil
&&
db
.
startPos
!=
nil
&&
db
.
startPos
.
Timestamp
<
minTs
{
minTs
=
db
.
startPos
.
Timestamp
}
}
return
minTs
}
return
func
(
segments
[]
*
Segment
,
ts
Timestamp
,
_
*
atomic
.
Bool
)
[]
UniqueID
{
segmentsToSync
:=
make
([]
UniqueID
,
0
)
for
_
,
segment
:=
range
segments
{
segmentMinTs
:=
segmentMinTs
(
segment
)
segmentStartTime
:=
tsoutil
.
PhysicalTime
(
segmentMinTs
)
cpLagDuration
:=
tsoutil
.
PhysicalTime
(
ts
)
.
Sub
(
segmentStartTime
)
shouldSync
:=
cpLagDuration
>
Params
.
DataNodeCfg
.
CpLagPeriod
.
GetAsDuration
(
time
.
Second
)
&&
!
segment
.
isBufferEmpty
()
if
shouldSync
{
segmentsToSync
=
append
(
segmentsToSync
,
segment
.
segmentID
)
}
}
if
len
(
segmentsToSync
)
>
0
{
log
.
Info
(
"sync segment for cp lag behind too much"
,
zap
.
Int64s
(
"segmentID"
,
segmentsToSync
))
}
return
segmentsToSync
}
}
internal/datanode/segment_sync_policy_test.go
浏览文件 @
af1d84e5
...
...
@@ -21,6 +21,7 @@ import (
"testing"
"time"
"github.com/milvus-io/milvus-proto/go-api/v2/msgpb"
"github.com/milvus-io/milvus/pkg/util/tsoutil"
"github.com/stretchr/testify/assert"
"go.uber.org/atomic"
...
...
@@ -94,3 +95,70 @@ func TestSyncMemoryTooHigh(t *testing.T) {
})
}
}
func
TestSyncCpLagBehindTooMuch
(
t
*
testing
.
T
)
{
nowTs
:=
tsoutil
.
ComposeTSByTime
(
time
.
Now
(),
0
)
laggedTs
:=
tsoutil
.
AddPhysicalDurationOnTs
(
nowTs
,
-
2
*
Params
.
DataNodeCfg
.
CpLagPeriod
.
GetAsDuration
(
time
.
Second
))
tests
:=
[]
struct
{
testName
string
segments
[]
*
Segment
idsToSync
[]
int64
}{
{
"test_current_buf_lag_behind"
,
[]
*
Segment
{
{
segmentID
:
1
,
curInsertBuf
:
&
BufferData
{
startPos
:
&
msgpb
.
MsgPosition
{
Timestamp
:
laggedTs
,
},
},
},
{
segmentID
:
2
,
curDeleteBuf
:
&
DelDataBuf
{
startPos
:
&
msgpb
.
MsgPosition
{
Timestamp
:
laggedTs
,
},
},
},
},
[]
int64
{
1
,
2
},
},
{
"test_history_buf_lag_behind"
,
[]
*
Segment
{
{
segmentID
:
1
,
historyInsertBuf
:
[]
*
BufferData
{
{
startPos
:
&
msgpb
.
MsgPosition
{
Timestamp
:
laggedTs
,
},
},
},
},
{
segmentID
:
2
,
historyDeleteBuf
:
[]
*
DelDataBuf
{
{
startPos
:
&
msgpb
.
MsgPosition
{
Timestamp
:
laggedTs
,
},
},
},
},
{
segmentID
:
3
,
},
},
[]
int64
{
1
,
2
},
},
}
for
_
,
test
:=
range
tests
{
t
.
Run
(
test
.
testName
,
func
(
t
*
testing
.
T
)
{
policy
:=
syncCPLagTooBehind
()
ids
:=
policy
(
test
.
segments
,
tsoutil
.
ComposeTSByTime
(
time
.
Now
(),
0
),
nil
)
assert
.
ElementsMatch
(
t
,
test
.
idsToSync
,
ids
)
})
}
}
pkg/util/paramtable/component_param.go
浏览文件 @
af1d84e5
...
...
@@ -2138,6 +2138,7 @@ type dataNodeConfig struct {
FlushDeleteBufferBytes
ParamItem
`refreshable:"true"`
BinLogMaxSize
ParamItem
`refreshable:"true"`
SyncPeriod
ParamItem
`refreshable:"true"`
CpLagPeriod
ParamItem
`refreshable:"true"`
// watchEvent
WatchEventTicklerInterval
ParamItem
`refreshable:"false"`
...
...
@@ -2258,6 +2259,15 @@ func (p *dataNodeConfig) init(base *BaseTable) {
}
p
.
SyncPeriod
.
Init
(
base
.
mgr
)
p
.
CpLagPeriod
=
ParamItem
{
Key
:
"datanode.segment.cpLagPeriod"
,
Version
:
"2.2.0"
,
DefaultValue
:
"600"
,
Doc
:
"The period to sync segments if buffer is not empty."
,
Export
:
true
,
}
p
.
CpLagPeriod
.
Init
(
base
.
mgr
)
p
.
WatchEventTicklerInterval
=
ParamItem
{
Key
:
"datanode.segment.watchEventTicklerInterval"
,
Version
:
"2.2.3"
,
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录