Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
f16bbe71
M
milvus
项目概览
milvus
/
milvus
11 个月 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
f16bbe71
编写于
2月 06, 2023
作者:
J
jaime
提交者:
GitHub
2月 06, 2023
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Add a segment seal policy by number of binlog files (#21941)
Signed-off-by:
N
jaime
<
yun.zhang@zilliz.com
>
上级
68156b3d
变更
5
隐藏空白更改
内联
并排
Showing
5 changed file
with
112 addition
and
2 deletion
+112
-2
configs/milvus.yaml
configs/milvus.yaml
+3
-0
internal/datacoord/segment_allocation_policy.go
internal/datacoord/segment_allocation_policy.go
+13
-1
internal/datacoord/segment_manager.go
internal/datacoord/segment_manager.go
+1
-0
internal/datacoord/segment_manager_test.go
internal/datacoord/segment_manager_test.go
+88
-1
internal/util/paramtable/component_param.go
internal/util/paramtable/component_param.go
+7
-0
未找到文件。
configs/milvus.yaml
浏览文件 @
f16bbe71
...
...
@@ -288,6 +288,9 @@ dataCoord:
# `minSizeFromIdleToSealed`, Milvus will automatically seal it.
maxIdleTime
:
600
# The max idle time of segment in seconds, 10*60.
minSizeFromIdleToSealed
:
16
# The min size in MB of segment which can be idle from sealed.
# The max number of binlog file for one segment, the segment will be sealed if
# the number of binlog file reaches to max value.
maxBinlogFileNumber
:
16
smallProportion
:
0.5
# The segment is considered as "small segment" when its # of rows is smaller than
# (smallProportion * segment max # of rows).
compactableProportion
:
0.85
# A compaction will happen on small segments if the segment after compaction will have
...
...
internal/datacoord/segment_allocation_policy.go
浏览文件 @
f16bbe71
...
...
@@ -116,7 +116,7 @@ func getSegmentCapacityPolicy(sizeFactor float64) segmentSealPolicy {
}
}
//
getLastExpiresLifetim
ePolicy get segmentSealPolicy with lifetime limit compares ts - segment.lastExpireTime
//
sealByMaxBinlogSiz
ePolicy get segmentSealPolicy with lifetime limit compares ts - segment.lastExpireTime
func
sealByLifetimePolicy
(
lifetime
time
.
Duration
)
segmentSealPolicy
{
return
func
(
segment
*
SegmentInfo
,
ts
Timestamp
)
bool
{
pts
,
_
:=
tsoutil
.
ParseTS
(
ts
)
...
...
@@ -126,6 +126,18 @@ func sealByLifetimePolicy(lifetime time.Duration) segmentSealPolicy {
}
}
// sealByMaxBinlogSizePolicy seal segment if binlog file number of segment exceed configured max number
func
sealByMaxBinlogFileNumberPolicy
(
maxBinlogFileNumber
int
)
segmentSealPolicy
{
return
func
(
segment
*
SegmentInfo
,
ts
Timestamp
)
bool
{
logFileCounter
:=
0
for
_
,
fieldBinlog
:=
range
segment
.
GetStatslogs
()
{
logFileCounter
+=
len
(
fieldBinlog
.
GetBinlogs
())
}
return
logFileCounter
>=
maxBinlogFileNumber
}
}
// sealLongTimeIdlePolicy seal segment if the segment has been written with a high frequency before.
// serve for this case:
// If users insert entities into segment continuously within a certain period of time, but they forgot to flush/(seal)
...
...
internal/datacoord/segment_manager.go
浏览文件 @
f16bbe71
...
...
@@ -184,6 +184,7 @@ func defaultAllocatePolicy() AllocatePolicy {
func
defaultSegmentSealPolicy
()
[]
segmentSealPolicy
{
return
[]
segmentSealPolicy
{
sealByMaxBinlogFileNumberPolicy
(
Params
.
DataCoordCfg
.
SegmentMaxBinlogFileNumber
),
sealByLifetimePolicy
(
Params
.
DataCoordCfg
.
SegmentMaxLifetime
),
getSegmentCapacityPolicy
(
Params
.
DataCoordCfg
.
SegmentSealProportion
),
sealLongTimeIdlePolicy
(
Params
.
DataCoordCfg
.
SegmentMaxIdleTime
,
Params
.
DataCoordCfg
.
SegmentMinSizeFromIdleToSealed
,
Params
.
DataCoordCfg
.
SegmentMaxSize
),
...
...
internal/datacoord/segment_manager_test.go
浏览文件 @
f16bbe71
...
...
@@ -23,11 +23,13 @@ import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/metastore/kv/datacoord"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/
stretchr/testify/assert
"
"github.com/
milvus-io/milvus/internal/util/metautil
"
)
func
TestManagerOptions
(
t
*
testing
.
T
)
{
...
...
@@ -517,6 +519,91 @@ func TestTryToSealSegment(t *testing.T) {
}
})
t
.
Run
(
"test sealByMaxBinlogFileNumberPolicy"
,
func
(
t
*
testing
.
T
)
{
Params
.
Init
()
mockAllocator
:=
newMockAllocator
()
meta
,
err
:=
newMemoryMeta
()
assert
.
Nil
(
t
,
err
)
schema
:=
newTestSchema
()
collID
,
err
:=
mockAllocator
.
allocID
(
context
.
Background
())
assert
.
Nil
(
t
,
err
)
meta
.
AddCollection
(
&
collectionInfo
{
ID
:
collID
,
Schema
:
schema
})
segmentManager
:=
newSegmentManager
(
meta
,
mockAllocator
,
nil
)
allocations
,
err
:=
segmentManager
.
AllocSegment
(
context
.
TODO
(),
collID
,
0
,
"c1"
,
2
)
assert
.
Nil
(
t
,
err
)
assert
.
EqualValues
(
t
,
1
,
len
(
allocations
))
ts
,
err
:=
segmentManager
.
allocator
.
allocTimestamp
(
context
.
Background
())
assert
.
Nil
(
t
,
err
)
// No seal polices
{
err
=
segmentManager
.
tryToSealSegment
(
ts
,
"c1"
)
assert
.
Nil
(
t
,
err
)
segments
:=
segmentManager
.
meta
.
segments
.
segments
assert
.
Equal
(
t
,
1
,
len
(
segments
))
for
_
,
seg
:=
range
segments
{
assert
.
Equal
(
t
,
commonpb
.
SegmentState_Growing
,
seg
.
GetState
())
}
}
// Not trigger seal
{
segmentManager
.
segmentSealPolicies
=
[]
segmentSealPolicy
{
sealByMaxBinlogFileNumberPolicy
(
2
)}
segments
:=
segmentManager
.
meta
.
segments
.
segments
assert
.
Equal
(
t
,
1
,
len
(
segments
))
for
_
,
seg
:=
range
segments
{
seg
.
Statslogs
=
[]
*
datapb
.
FieldBinlog
{
{
FieldID
:
2
,
Binlogs
:
[]
*
datapb
.
Binlog
{
{
EntriesNum
:
10
,
LogID
:
3
,
LogPath
:
metautil
.
BuildInsertLogPath
(
""
,
1
,
1
,
seg
.
ID
,
2
,
3
),
},
},
},
}
err
=
segmentManager
.
tryToSealSegment
(
ts
,
"c1"
)
assert
.
Nil
(
t
,
err
)
seg
=
segmentManager
.
meta
.
segments
.
segments
[
seg
.
ID
]
assert
.
Equal
(
t
,
commonpb
.
SegmentState_Growing
,
seg
.
GetState
())
}
}
// Trigger seal
{
segmentManager
.
segmentSealPolicies
=
[]
segmentSealPolicy
{
sealByMaxBinlogFileNumberPolicy
(
2
)}
segments
:=
segmentManager
.
meta
.
segments
.
segments
assert
.
Equal
(
t
,
1
,
len
(
segments
))
for
_
,
seg
:=
range
segments
{
seg
.
Statslogs
=
[]
*
datapb
.
FieldBinlog
{
{
FieldID
:
1
,
Binlogs
:
[]
*
datapb
.
Binlog
{
{
EntriesNum
:
10
,
LogID
:
1
,
LogPath
:
metautil
.
BuildInsertLogPath
(
""
,
1
,
1
,
seg
.
ID
,
1
,
3
),
},
{
EntriesNum
:
20
,
LogID
:
2
,
LogPath
:
metautil
.
BuildInsertLogPath
(
""
,
1
,
1
,
seg
.
ID
,
1
,
2
),
},
},
},
}
err
=
segmentManager
.
tryToSealSegment
(
ts
,
"c1"
)
assert
.
Nil
(
t
,
err
)
seg
=
segmentManager
.
meta
.
segments
.
segments
[
seg
.
ID
]
assert
.
Equal
(
t
,
commonpb
.
SegmentState_Sealed
,
seg
.
GetState
())
}
}
})
t
.
Run
(
"seal with segment policy with kv fails"
,
func
(
t
*
testing
.
T
)
{
Params
.
Init
()
mockAllocator
:=
newMockAllocator
()
...
...
internal/util/paramtable/component_param.go
浏览文件 @
f16bbe71
...
...
@@ -1278,6 +1278,7 @@ type dataCoordConfig struct {
SegmentMaxLifetime
time
.
Duration
SegmentMaxIdleTime
time
.
Duration
SegmentMinSizeFromIdleToSealed
float64
SegmentMaxBinlogFileNumber
int
CreatedTime
time
.
Time
UpdatedTime
time
.
Time
...
...
@@ -1319,6 +1320,7 @@ func (p *dataCoordConfig) init(base *BaseTable) {
p
.
initSegmentMaxLifetime
()
p
.
initSegmentMaxIdleTime
()
p
.
initSegmentMinSizeFromIdleToSealed
()
p
.
initSegmentMaxBinlogFileNumber
()
p
.
initEnableCompaction
()
p
.
initEnableAutoCompaction
()
...
...
@@ -1375,6 +1377,11 @@ func (p *dataCoordConfig) initSegmentMinSizeFromIdleToSealed() {
log
.
Info
(
"init segment min size from idle to sealed"
,
zap
.
Float64
(
"value"
,
p
.
SegmentMinSizeFromIdleToSealed
))
}
func
(
p
*
dataCoordConfig
)
initSegmentMaxBinlogFileNumber
()
{
p
.
SegmentMaxBinlogFileNumber
=
p
.
Base
.
ParseIntWithDefault
(
"dataCoord.segment.maxBinlogFileNumber"
,
16
)
log
.
Info
(
"init segment max binlog file to sealed"
,
zap
.
Int
(
"value"
,
p
.
SegmentMaxBinlogFileNumber
))
}
func
(
p
*
dataCoordConfig
)
initChannelWatchPrefix
()
{
// WARN: this value should not be put to milvus.yaml. It's a default value for channel watch path.
// This will be removed after we reconstruct our config module.
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录