Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
5b22697e
M
milvus
项目概览
milvus
/
milvus
11 个月 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
5b22697e
编写于
9月 07, 2021
作者:
C
congqixia
提交者:
GitHub
9月 07, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Add segment manager unit tests (#7525)
Signed-off-by:
N
Congqi Xia
<
congqi.xia@zilliz.com
>
上级
32650570
变更
4
隐藏空白更改
内联
并排
Showing
4 changed file
with
313 addition
and
39 deletion
+313
-39
internal/datacoord/mock_test.go
internal/datacoord/mock_test.go
+12
-0
internal/datacoord/segment_allocation_policy.go
internal/datacoord/segment_allocation_policy.go
+7
-10
internal/datacoord/segment_manager.go
internal/datacoord/segment_manager.go
+33
-29
internal/datacoord/segment_manager_test.go
internal/datacoord/segment_manager_test.go
+261
-0
未找到文件。
internal/datacoord/mock_test.go
浏览文件 @
5b22697e
...
...
@@ -12,6 +12,7 @@ package datacoord
import
(
"context"
"errors"
"sync/atomic"
"time"
...
...
@@ -53,6 +54,17 @@ func (m *MockAllocator) allocID(ctx context.Context) (UniqueID, error) {
return
val
,
nil
}
// FailsAllocator allocator that fails
type
FailsAllocator
struct
{}
func
(
a
*
FailsAllocator
)
allocTimestamp
(
_
context
.
Context
)
(
Timestamp
,
error
)
{
return
0
,
errors
.
New
(
"always fail"
)
}
func
(
a
*
FailsAllocator
)
allocID
(
_
context
.
Context
)
(
UniqueID
,
error
)
{
return
0
,
errors
.
New
(
"always fail"
)
}
func
newMockAllocator
()
*
MockAllocator
{
return
&
MockAllocator
{}
}
...
...
internal/datacoord/segment_allocation_policy.go
浏览文件 @
5b22697e
...
...
@@ -49,9 +49,7 @@ func AllocatePolicyV1(segments []*SegmentInfo, count int64,
existedSegmentAllocations
:=
make
([]
*
Allocation
,
0
)
// create new segment if count >= max num
for
count
>=
maxCountPerSegment
{
allocation
:=
&
Allocation
{
NumOfRows
:
maxCountPerSegment
,
}
allocation
:=
getAllocation
(
maxCountPerSegment
)
newSegmentAllocations
=
append
(
newSegmentAllocations
,
allocation
)
count
-=
maxCountPerSegment
}
...
...
@@ -69,18 +67,14 @@ func AllocatePolicyV1(segments []*SegmentInfo, count int64,
if
free
<
count
{
continue
}
allocation
:=
&
Allocation
{
SegmentID
:
segment
.
GetID
(),
NumOfRows
:
count
,
}
allocation
:=
getAllocation
(
count
)
allocation
.
SegmentID
=
segment
.
GetID
()
existedSegmentAllocations
=
append
(
existedSegmentAllocations
,
allocation
)
return
newSegmentAllocations
,
existedSegmentAllocations
}
// allocate new segment for remaining count
allocation
:=
&
Allocation
{
NumOfRows
:
count
,
}
allocation
:=
getAllocation
(
count
)
newSegmentAllocations
=
append
(
newSegmentAllocations
,
allocation
)
return
newSegmentAllocations
,
existedSegmentAllocations
}
...
...
@@ -122,6 +116,9 @@ func getChannelOpenSegCapacityPolicy(limit int) channelSealPolicy {
}
sortSegmentsByLastExpires
(
segs
)
offLen
:=
len
(
segs
)
-
limit
if
offLen
>
len
(
segs
)
{
offLen
=
len
(
segs
)
}
return
segs
[
0
:
offLen
]
}
}
...
...
internal/datacoord/segment_manager.go
浏览文件 @
5b22697e
...
...
@@ -28,6 +28,37 @@ import (
"github.com/milvus-io/milvus/internal/proto/datapb"
)
var
(
allocPool
=
sync
.
Pool
{
New
:
func
()
interface
{}
{
return
&
Allocation
{}
},
}
)
// getAllocation unified way to retrieve allocation struct
func
getAllocation
(
numOfRows
int64
)
*
Allocation
{
v
:=
allocPool
.
Get
()
a
,
ok
:=
v
.
(
*
Allocation
)
if
!
ok
{
a
=
&
Allocation
{}
}
if
a
==
nil
{
return
&
Allocation
{
NumOfRows
:
numOfRows
,
}
}
a
.
NumOfRows
=
numOfRows
a
.
ExpireTime
=
0
a
.
SegmentID
=
0
return
a
}
// putAllocation put allocation for recycling
func
putAllocation
(
a
*
Allocation
)
{
allocPool
.
Put
(
a
)
}
const
segmentMaxLifetime
=
24
*
time
.
Hour
// Manager manage segment related operations.
...
...
@@ -63,7 +94,6 @@ type SegmentManager struct {
segmentSealPolicies
[]
segmentSealPolicy
channelSealPolicies
[]
channelSealPolicy
flushPolicy
flushPolicy
allocPool
sync
.
Pool
}
type
allocHelper
struct
{
...
...
@@ -157,11 +187,6 @@ func newSegmentManager(meta *meta, allocator allocator, opts ...allocOption) *Se
segmentSealPolicies
:
defaultSegmentSealPolicy
(),
// default only segment size policy
channelSealPolicies
:
[]
channelSealPolicy
{},
// no default channel seal policy
flushPolicy
:
defaultFlushPolicy
(),
allocPool
:
sync
.
Pool
{
New
:
func
()
interface
{}
{
return
&
Allocation
{}
},
},
}
for
_
,
opt
:=
range
opts
{
opt
.
apply
(
manager
)
...
...
@@ -180,27 +205,6 @@ func (s *SegmentManager) loadSegmentsFromMeta() {
s
.
segments
=
segmentsID
}
// getAllocation unified way to retrieve allocation struct
func
(
s
*
SegmentManager
)
getAllocation
(
numOfRows
int64
)
*
Allocation
{
v
:=
s
.
allocPool
.
Get
()
if
v
==
nil
{
return
&
Allocation
{
NumOfRows
:
numOfRows
,
}
}
a
,
ok
:=
v
.
(
*
Allocation
)
if
!
ok
{
a
=
&
Allocation
{}
}
a
.
NumOfRows
=
numOfRows
return
a
}
// putAllocation put allocation for recycling
func
(
s
*
SegmentManager
)
putAllocation
(
a
*
Allocation
)
{
s
.
allocPool
.
Put
(
a
)
}
// AllocSegment allocate segment per request collcation, partication, channel and rows
func
(
s
*
SegmentManager
)
AllocSegment
(
ctx
context
.
Context
,
collectionID
UniqueID
,
partitionID
UniqueID
,
channelName
string
,
requestRows
int64
)
([]
*
Allocation
,
error
)
{
...
...
@@ -339,7 +343,7 @@ func (s *SegmentManager) DropSegment(ctx context.Context, segmentID UniqueID) {
}
s
.
meta
.
SetAllocations
(
segmentID
,
[]
*
Allocation
{})
for
_
,
allocation
:=
range
segment
.
allocations
{
s
.
putAllocation
(
allocation
)
putAllocation
(
allocation
)
}
}
...
...
@@ -407,7 +411,7 @@ func (s *SegmentManager) ExpireAllocations(channel string, ts Timestamp) error {
for
i
:=
0
;
i
<
len
(
segment
.
allocations
);
i
++
{
if
segment
.
allocations
[
i
]
.
ExpireTime
<=
ts
{
a
:=
segment
.
allocations
[
i
]
s
.
putAllocation
(
a
)
putAllocation
(
a
)
}
else
{
allocations
=
append
(
allocations
,
segment
.
allocations
[
i
])
}
...
...
internal/datacoord/segment_manager_test.go
浏览文件 @
5b22697e
...
...
@@ -11,9 +11,15 @@ package datacoord
import
(
"context"
"errors"
"fmt"
"math"
"sync"
"testing"
"time"
"github.com/milvus-io/milvus/internal/kv"
memkv
"github.com/milvus-io/milvus/internal/kv/mem"
"github.com/milvus-io/milvus/internal/proto/commonpb"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/schemapb"
...
...
@@ -21,6 +27,67 @@ import (
"github.com/stretchr/testify/assert"
)
func
TestManagerOptions
(
t
*
testing
.
T
)
{
// ctx := context.Background()
Params
.
Init
()
mockAllocator
:=
newMockAllocator
()
meta
,
err
:=
newMemoryMeta
(
mockAllocator
)
assert
.
Nil
(
t
,
err
)
segmentManager
:=
newSegmentManager
(
meta
,
mockAllocator
)
t
.
Run
(
"test with alloc helper"
,
func
(
t
*
testing
.
T
)
{
opt
:=
withAllocHelper
(
allocHelper
{})
opt
.
apply
(
segmentManager
)
assert
.
True
(
t
,
segmentManager
.
helper
.
afterCreateSegment
==
nil
)
})
t
.
Run
(
"test withCalUpperLimitPolicy"
,
func
(
t
*
testing
.
T
)
{
opt
:=
withCalUpperLimitPolicy
(
defaultCalUpperLimitPolicy
())
assert
.
NotNil
(
t
,
opt
)
//manual set nil``
segmentManager
.
estimatePolicy
=
nil
opt
.
apply
(
segmentManager
)
assert
.
True
(
t
,
segmentManager
.
estimatePolicy
!=
nil
)
})
t
.
Run
(
"test withAllocPolicy"
,
func
(
t
*
testing
.
T
)
{
opt
:=
withAllocPolicy
(
defaultAlocatePolicy
())
assert
.
NotNil
(
t
,
opt
)
// manual set nil
segmentManager
.
allocPolicy
=
nil
opt
.
apply
(
segmentManager
)
assert
.
True
(
t
,
segmentManager
.
allocPolicy
!=
nil
)
})
t
.
Run
(
"test withSegmentSealPolicy"
,
func
(
t
*
testing
.
T
)
{
opt
:=
withSegmentSealPolices
(
defaultSegmentSealPolicy
()
...
)
assert
.
NotNil
(
t
,
opt
)
// manual set nil
segmentManager
.
segmentSealPolicies
=
[]
segmentSealPolicy
{}
opt
.
apply
(
segmentManager
)
assert
.
True
(
t
,
len
(
segmentManager
.
segmentSealPolicies
)
>
0
)
})
t
.
Run
(
"test withChannelSealPolicies"
,
func
(
t
*
testing
.
T
)
{
opt
:=
withChannelSealPolices
(
getChannelOpenSegCapacityPolicy
(
1000
))
assert
.
NotNil
(
t
,
opt
)
// manaul set nil
segmentManager
.
channelSealPolicies
=
[]
channelSealPolicy
{}
opt
.
apply
(
segmentManager
)
assert
.
True
(
t
,
len
(
segmentManager
.
channelSealPolicies
)
>
0
)
})
t
.
Run
(
"test withFlushPolicy"
,
func
(
t
*
testing
.
T
)
{
opt
:=
withFlushPolicy
(
defaultFlushPolicy
())
assert
.
NotNil
(
t
,
opt
)
// manual set nil
segmentManager
.
flushPolicy
=
nil
opt
.
apply
(
segmentManager
)
assert
.
True
(
t
,
segmentManager
.
flushPolicy
!=
nil
)
})
}
func
TestAllocSegment
(
t
*
testing
.
T
)
{
ctx
:=
context
.
Background
()
Params
.
Init
()
...
...
@@ -42,6 +109,13 @@ func TestAllocSegment(t *testing.T) {
assert
.
NotEqualValues
(
t
,
0
,
allocations
[
0
]
.
SegmentID
)
assert
.
NotEqualValues
(
t
,
0
,
allocations
[
0
]
.
ExpireTime
)
})
t
.
Run
(
"allocation fails"
,
func
(
t
*
testing
.
T
)
{
failsAllocator
:=
&
FailsAllocator
{}
segmentManager
:=
newSegmentManager
(
meta
,
failsAllocator
)
_
,
err
:=
segmentManager
.
AllocSegment
(
ctx
,
collID
,
100
,
"c1"
,
100
)
assert
.
NotNil
(
t
,
err
)
})
}
func
TestLoadSegmentsFromMeta
(
t
*
testing
.
T
)
{
...
...
@@ -241,3 +315,190 @@ func TestGetFlushableSegments(t *testing.T) {
assert
.
EqualValues
(
t
,
allocations
[
0
]
.
SegmentID
,
ids
[
0
])
})
}
// a mock kv that always fail when do `Save`
type
saveFailKv
struct
{
kv
.
TxnKV
}
// LoadWithPrefix override behavior
func
(
kv
*
saveFailKv
)
Save
(
key
,
value
string
)
error
{
fmt
.
Println
(
"here"
)
return
errors
.
New
(
"mocked fail"
)
}
func
TestTryToSealSegment
(
t
*
testing
.
T
)
{
t
.
Run
(
"normal seal with segment policies"
,
func
(
t
*
testing
.
T
)
{
Params
.
Init
()
mockAllocator
:=
newMockAllocator
()
meta
,
err
:=
newMemoryMeta
(
mockAllocator
)
assert
.
Nil
(
t
,
err
)
schema
:=
newTestSchema
()
collID
,
err
:=
mockAllocator
.
allocID
(
context
.
Background
())
assert
.
Nil
(
t
,
err
)
meta
.
AddCollection
(
&
datapb
.
CollectionInfo
{
ID
:
collID
,
Schema
:
schema
})
segmentManager
:=
newSegmentManager
(
meta
,
mockAllocator
,
withSegmentSealPolices
(
sealByLifetimePolicy
(
math
.
MinInt64
)))
//always seal
allocations
,
err
:=
segmentManager
.
AllocSegment
(
context
.
TODO
(),
collID
,
0
,
"c1"
,
2
)
assert
.
Nil
(
t
,
err
)
assert
.
EqualValues
(
t
,
1
,
len
(
allocations
))
ts
,
err
:=
segmentManager
.
allocator
.
allocTimestamp
(
context
.
Background
())
assert
.
Nil
(
t
,
err
)
err
=
segmentManager
.
tryToSealSegment
(
ts
,
"c1"
)
assert
.
Nil
(
t
,
err
)
for
_
,
seg
:=
range
segmentManager
.
meta
.
segments
.
segments
{
assert
.
Equal
(
t
,
commonpb
.
SegmentState_Sealed
,
seg
.
GetState
())
}
})
t
.
Run
(
"normal seal with channel seal policies"
,
func
(
t
*
testing
.
T
)
{
Params
.
Init
()
mockAllocator
:=
newMockAllocator
()
meta
,
err
:=
newMemoryMeta
(
mockAllocator
)
assert
.
Nil
(
t
,
err
)
schema
:=
newTestSchema
()
collID
,
err
:=
mockAllocator
.
allocID
(
context
.
Background
())
assert
.
Nil
(
t
,
err
)
meta
.
AddCollection
(
&
datapb
.
CollectionInfo
{
ID
:
collID
,
Schema
:
schema
})
segmentManager
:=
newSegmentManager
(
meta
,
mockAllocator
,
withChannelSealPolices
(
getChannelOpenSegCapacityPolicy
(
-
1
)))
//always seal
allocations
,
err
:=
segmentManager
.
AllocSegment
(
context
.
TODO
(),
collID
,
0
,
"c1"
,
2
)
assert
.
Nil
(
t
,
err
)
assert
.
EqualValues
(
t
,
1
,
len
(
allocations
))
ts
,
err
:=
segmentManager
.
allocator
.
allocTimestamp
(
context
.
Background
())
assert
.
Nil
(
t
,
err
)
err
=
segmentManager
.
tryToSealSegment
(
ts
,
"c1"
)
assert
.
Nil
(
t
,
err
)
for
_
,
seg
:=
range
segmentManager
.
meta
.
segments
.
segments
{
assert
.
Equal
(
t
,
commonpb
.
SegmentState_Sealed
,
seg
.
GetState
())
}
})
t
.
Run
(
"normal seal with both segment & channel seal policy"
,
func
(
t
*
testing
.
T
)
{
Params
.
Init
()
mockAllocator
:=
newMockAllocator
()
meta
,
err
:=
newMemoryMeta
(
mockAllocator
)
assert
.
Nil
(
t
,
err
)
schema
:=
newTestSchema
()
collID
,
err
:=
mockAllocator
.
allocID
(
context
.
Background
())
assert
.
Nil
(
t
,
err
)
meta
.
AddCollection
(
&
datapb
.
CollectionInfo
{
ID
:
collID
,
Schema
:
schema
})
segmentManager
:=
newSegmentManager
(
meta
,
mockAllocator
,
withSegmentSealPolices
(
sealByLifetimePolicy
(
math
.
MinInt64
)),
withChannelSealPolices
(
getChannelOpenSegCapacityPolicy
(
-
1
)))
//always seal
allocations
,
err
:=
segmentManager
.
AllocSegment
(
context
.
TODO
(),
collID
,
0
,
"c1"
,
2
)
assert
.
Nil
(
t
,
err
)
assert
.
EqualValues
(
t
,
1
,
len
(
allocations
))
ts
,
err
:=
segmentManager
.
allocator
.
allocTimestamp
(
context
.
Background
())
assert
.
Nil
(
t
,
err
)
err
=
segmentManager
.
tryToSealSegment
(
ts
,
"c1"
)
assert
.
Nil
(
t
,
err
)
for
_
,
seg
:=
range
segmentManager
.
meta
.
segments
.
segments
{
assert
.
Equal
(
t
,
commonpb
.
SegmentState_Sealed
,
seg
.
GetState
())
}
})
t
.
Run
(
"seal with segment policy with kv fails"
,
func
(
t
*
testing
.
T
)
{
Params
.
Init
()
mockAllocator
:=
newMockAllocator
()
memoryKV
:=
memkv
.
NewMemoryKV
()
fkv
:=
&
saveFailKv
{
TxnKV
:
memoryKV
}
meta
,
err
:=
NewMeta
(
memoryKV
)
assert
.
Nil
(
t
,
err
)
schema
:=
newTestSchema
()
collID
,
err
:=
mockAllocator
.
allocID
(
context
.
Background
())
assert
.
Nil
(
t
,
err
)
meta
.
AddCollection
(
&
datapb
.
CollectionInfo
{
ID
:
collID
,
Schema
:
schema
})
segmentManager
:=
newSegmentManager
(
meta
,
mockAllocator
,
withSegmentSealPolices
(
sealByLifetimePolicy
(
math
.
MinInt64
)))
//always seal
allocations
,
err
:=
segmentManager
.
AllocSegment
(
context
.
TODO
(),
collID
,
0
,
"c1"
,
2
)
assert
.
Nil
(
t
,
err
)
assert
.
EqualValues
(
t
,
1
,
len
(
allocations
))
segmentManager
.
meta
.
client
=
fkv
ts
,
err
:=
segmentManager
.
allocator
.
allocTimestamp
(
context
.
Background
())
assert
.
Nil
(
t
,
err
)
err
=
segmentManager
.
tryToSealSegment
(
ts
,
"c1"
)
assert
.
NotNil
(
t
,
err
)
})
t
.
Run
(
"seal with channel policy with kv fails"
,
func
(
t
*
testing
.
T
)
{
Params
.
Init
()
mockAllocator
:=
newMockAllocator
()
memoryKV
:=
memkv
.
NewMemoryKV
()
fkv
:=
&
saveFailKv
{
TxnKV
:
memoryKV
}
meta
,
err
:=
NewMeta
(
memoryKV
)
assert
.
Nil
(
t
,
err
)
schema
:=
newTestSchema
()
collID
,
err
:=
mockAllocator
.
allocID
(
context
.
Background
())
assert
.
Nil
(
t
,
err
)
meta
.
AddCollection
(
&
datapb
.
CollectionInfo
{
ID
:
collID
,
Schema
:
schema
})
segmentManager
:=
newSegmentManager
(
meta
,
mockAllocator
,
withChannelSealPolices
(
getChannelOpenSegCapacityPolicy
(
-
1
)))
//always seal
allocations
,
err
:=
segmentManager
.
AllocSegment
(
context
.
TODO
(),
collID
,
0
,
"c1"
,
2
)
assert
.
Nil
(
t
,
err
)
assert
.
EqualValues
(
t
,
1
,
len
(
allocations
))
segmentManager
.
meta
.
client
=
fkv
ts
,
err
:=
segmentManager
.
allocator
.
allocTimestamp
(
context
.
Background
())
assert
.
Nil
(
t
,
err
)
err
=
segmentManager
.
tryToSealSegment
(
ts
,
"c1"
)
assert
.
NotNil
(
t
,
err
)
})
}
func
TestAllocationPool
(
t
*
testing
.
T
)
{
t
.
Run
(
"normal get&put"
,
func
(
t
*
testing
.
T
)
{
allocPool
=
sync
.
Pool
{
New
:
func
()
interface
{}
{
return
&
Allocation
{}
},
}
allo
:=
getAllocation
(
100
)
assert
.
EqualValues
(
t
,
100
,
allo
.
NumOfRows
)
assert
.
EqualValues
(
t
,
0
,
allo
.
ExpireTime
)
assert
.
EqualValues
(
t
,
0
,
allo
.
SegmentID
)
putAllocation
(
allo
)
})
t
.
Run
(
"put nil"
,
func
(
t
*
testing
.
T
)
{
var
allo
*
Allocation
=
nil
allocPool
=
sync
.
Pool
{
New
:
func
()
interface
{}
{
return
&
Allocation
{}
},
}
putAllocation
(
allo
)
allo
=
getAllocation
(
100
)
assert
.
EqualValues
(
t
,
100
,
allo
.
NumOfRows
)
assert
.
EqualValues
(
t
,
0
,
allo
.
ExpireTime
)
assert
.
EqualValues
(
t
,
0
,
allo
.
SegmentID
)
})
t
.
Run
(
"put something else"
,
func
(
t
*
testing
.
T
)
{
allocPool
=
sync
.
Pool
{
New
:
func
()
interface
{}
{
return
&
Allocation
{}
},
}
allocPool
.
Put
(
&
struct
{}{})
allo
:=
getAllocation
(
100
)
assert
.
EqualValues
(
t
,
100
,
allo
.
NumOfRows
)
assert
.
EqualValues
(
t
,
0
,
allo
.
ExpireTime
)
assert
.
EqualValues
(
t
,
0
,
allo
.
SegmentID
)
})
}
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录