Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
d4016088
M
milvus
项目概览
milvus
/
milvus
大约 1 年 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
d4016088
编写于
12月 20, 2022
作者:
J
jaime
提交者:
GitHub
12月 20, 2022
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Add a segment seal policy by number of binlog files (#21263)
Signed-off-by:
N
yun.zhang
<
yun.zhang@zilliz.com
>
上级
856bceec
变更
5
隐藏空白更改
内联
并排
Showing
5 changed file
with
113 addition
and
2 deletion
+113
-2
configs/milvus.yaml
configs/milvus.yaml
+3
-0
internal/datacoord/segment_allocation_policy.go
internal/datacoord/segment_allocation_policy.go
+13
-1
internal/datacoord/segment_manager.go
internal/datacoord/segment_manager.go
+1
-0
internal/datacoord/segment_manager_test.go
internal/datacoord/segment_manager_test.go
+88
-1
internal/util/paramtable/component_param.go
internal/util/paramtable/component_param.go
+8
-0
未找到文件。
configs/milvus.yaml
浏览文件 @
d4016088
...
...
@@ -283,6 +283,9 @@ dataCoord:
# `minSizeFromIdleToSealed`, Milvus will automatically seal it.
maxIdleTime
:
600
# The max idle time of segment in seconds, 10*60.
minSizeFromIdleToSealed
:
16
# The min size in MB of segment which can be idle from sealed.
# The max number of binlog file for one segment, the segment will be sealed if
# the number of binlog file reaches to max value.
maxBinlogFileNumber
:
256
smallProportion
:
0.5
# The segment is considered as "small segment" when its # of rows is smaller than
# (smallProportion * segment max # of rows).
compactableProportion
:
0.5
# A compaction will happen on small segments if the segment after compaction will have
...
...
internal/datacoord/segment_allocation_policy.go
浏览文件 @
d4016088
...
...
@@ -116,7 +116,7 @@ func getSegmentCapacityPolicy(sizeFactor float64) segmentSealPolicy {
}
}
//
getLastExpiresLifetim
ePolicy get segmentSealPolicy with lifetime limit compares ts - segment.lastExpireTime
//
sealByMaxBinlogSiz
ePolicy get segmentSealPolicy with lifetime limit compares ts - segment.lastExpireTime
func
sealByLifetimePolicy
(
lifetime
time
.
Duration
)
segmentSealPolicy
{
return
func
(
segment
*
SegmentInfo
,
ts
Timestamp
)
bool
{
pts
,
_
:=
tsoutil
.
ParseTS
(
ts
)
...
...
@@ -126,6 +126,18 @@ func sealByLifetimePolicy(lifetime time.Duration) segmentSealPolicy {
}
}
// sealByMaxBinlogSizePolicy seal segment if binlog file number of segment exceed configured max number
func
sealByMaxBinlogFileNumberPolicy
(
maxBinlogFileNumber
int
)
segmentSealPolicy
{
return
func
(
segment
*
SegmentInfo
,
ts
Timestamp
)
bool
{
logFileCounter
:=
0
for
_
,
fieldBinlog
:=
range
segment
.
Binlogs
{
logFileCounter
+=
len
(
fieldBinlog
.
GetBinlogs
())
}
return
logFileCounter
>=
maxBinlogFileNumber
}
}
// sealLongTimeIdlePolicy seal segment if the segment has been written with a high frequency before.
// serve for this case:
// If users insert entities into segment continuously within a certain period of time, but they forgot to flush/(seal)
...
...
internal/datacoord/segment_manager.go
浏览文件 @
d4016088
...
...
@@ -185,6 +185,7 @@ func defaultAllocatePolicy() AllocatePolicy {
func
defaultSegmentSealPolicy
()
[]
segmentSealPolicy
{
return
[]
segmentSealPolicy
{
sealByMaxBinlogFileNumberPolicy
(
Params
.
DataCoordCfg
.
SegmentMaxBinlogFileNumber
.
GetAsInt
()),
sealByLifetimePolicy
(
Params
.
DataCoordCfg
.
SegmentMaxLifetime
.
GetAsDuration
(
time
.
Second
)),
getSegmentCapacityPolicy
(
Params
.
DataCoordCfg
.
SegmentSealProportion
.
GetAsFloat
()),
sealLongTimeIdlePolicy
(
Params
.
DataCoordCfg
.
SegmentMaxIdleTime
.
GetAsDuration
(
time
.
Second
),
Params
.
DataCoordCfg
.
SegmentMinSizeFromIdleToSealed
.
GetAsFloat
(),
Params
.
DataCoordCfg
.
SegmentMaxSize
.
GetAsFloat
()),
...
...
internal/datacoord/segment_manager_test.go
浏览文件 @
d4016088
...
...
@@ -23,12 +23,14 @@ import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
memkv
"github.com/milvus-io/milvus/internal/kv/mem"
"github.com/milvus-io/milvus/internal/metastore/kv/datacoord"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/
stretchr/testify/assert
"
"github.com/
milvus-io/milvus/internal/util/metautil
"
)
func
TestManagerOptions
(
t
*
testing
.
T
)
{
...
...
@@ -487,6 +489,91 @@ func TestTryToSealSegment(t *testing.T) {
}
})
t
.
Run
(
"test sealByMaxBinlogFileNumberPolicy"
,
func
(
t
*
testing
.
T
)
{
Params
.
Init
()
mockAllocator
:=
newMockAllocator
()
meta
,
err
:=
newMemoryMeta
()
assert
.
Nil
(
t
,
err
)
schema
:=
newTestSchema
()
collID
,
err
:=
mockAllocator
.
allocID
(
context
.
Background
())
assert
.
Nil
(
t
,
err
)
meta
.
AddCollection
(
&
collectionInfo
{
ID
:
collID
,
Schema
:
schema
})
segmentManager
:=
newSegmentManager
(
meta
,
mockAllocator
,
nil
)
allocations
,
err
:=
segmentManager
.
AllocSegment
(
context
.
TODO
(),
collID
,
0
,
"c1"
,
2
)
assert
.
Nil
(
t
,
err
)
assert
.
EqualValues
(
t
,
1
,
len
(
allocations
))
ts
,
err
:=
segmentManager
.
allocator
.
allocTimestamp
(
context
.
Background
())
assert
.
Nil
(
t
,
err
)
// No seal polices
{
err
=
segmentManager
.
tryToSealSegment
(
ts
,
"c1"
)
assert
.
Nil
(
t
,
err
)
segments
:=
segmentManager
.
meta
.
segments
.
segments
assert
.
Equal
(
t
,
1
,
len
(
segments
))
for
_
,
seg
:=
range
segments
{
assert
.
Equal
(
t
,
commonpb
.
SegmentState_Growing
,
seg
.
GetState
())
}
}
// Not trigger seal
{
segmentManager
.
segmentSealPolicies
=
[]
segmentSealPolicy
{
sealByMaxBinlogFileNumberPolicy
(
2
)}
segments
:=
segmentManager
.
meta
.
segments
.
segments
assert
.
Equal
(
t
,
1
,
len
(
segments
))
for
_
,
seg
:=
range
segments
{
seg
.
Binlogs
=
[]
*
datapb
.
FieldBinlog
{
{
FieldID
:
2
,
Binlogs
:
[]
*
datapb
.
Binlog
{
{
EntriesNum
:
10
,
LogID
:
3
,
LogPath
:
metautil
.
BuildInsertLogPath
(
""
,
1
,
1
,
seg
.
ID
,
2
,
3
),
},
},
},
}
err
=
segmentManager
.
tryToSealSegment
(
ts
,
"c1"
)
assert
.
Nil
(
t
,
err
)
seg
=
segmentManager
.
meta
.
segments
.
segments
[
seg
.
ID
]
assert
.
Equal
(
t
,
commonpb
.
SegmentState_Growing
,
seg
.
GetState
())
}
}
// Trigger seal
{
segmentManager
.
segmentSealPolicies
=
[]
segmentSealPolicy
{
sealByMaxBinlogFileNumberPolicy
(
2
)}
segments
:=
segmentManager
.
meta
.
segments
.
segments
assert
.
Equal
(
t
,
1
,
len
(
segments
))
for
_
,
seg
:=
range
segments
{
seg
.
Binlogs
=
[]
*
datapb
.
FieldBinlog
{
{
FieldID
:
1
,
Binlogs
:
[]
*
datapb
.
Binlog
{
{
EntriesNum
:
10
,
LogID
:
1
,
LogPath
:
metautil
.
BuildInsertLogPath
(
""
,
1
,
1
,
seg
.
ID
,
1
,
3
),
},
{
EntriesNum
:
20
,
LogID
:
2
,
LogPath
:
metautil
.
BuildInsertLogPath
(
""
,
1
,
1
,
seg
.
ID
,
1
,
2
),
},
},
},
}
err
=
segmentManager
.
tryToSealSegment
(
ts
,
"c1"
)
assert
.
Nil
(
t
,
err
)
seg
=
segmentManager
.
meta
.
segments
.
segments
[
seg
.
ID
]
assert
.
Equal
(
t
,
commonpb
.
SegmentState_Sealed
,
seg
.
GetState
())
}
}
})
t
.
Run
(
"seal with segment policy with kv fails"
,
func
(
t
*
testing
.
T
)
{
Params
.
Init
()
mockAllocator
:=
newMockAllocator
()
...
...
internal/util/paramtable/component_param.go
浏览文件 @
d4016088
...
...
@@ -1298,6 +1298,7 @@ type dataCoordConfig struct {
SegmentMaxLifetime
ParamItem
`refreshable:"false"`
SegmentMaxIdleTime
ParamItem
`refreshable:"false"`
SegmentMinSizeFromIdleToSealed
ParamItem
`refreshable:"false"`
SegmentMaxBinlogFileNumber
ParamItem
`refreshable:"false"`
// compaction
EnableCompaction
ParamItem
`refreshable:"false"`
...
...
@@ -1382,6 +1383,13 @@ func (p *dataCoordConfig) init(base *BaseTable) {
}
p
.
SegmentMinSizeFromIdleToSealed
.
Init
(
base
.
mgr
)
p
.
SegmentMaxBinlogFileNumber
=
ParamItem
{
Key
:
"dataCoord.segment.maxBinlogFileNumber"
,
Version
:
"2.0.0"
,
DefaultValue
:
"256"
,
}
p
.
SegmentMaxBinlogFileNumber
.
Init
(
base
.
mgr
)
p
.
EnableCompaction
=
ParamItem
{
Key
:
"dataCoord.enableCompaction"
,
Version
:
"2.0.0"
,
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录