Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
e3e3ac15
M
milvus
项目概览
milvus
/
milvus
大约 1 年 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
e3e3ac15
编写于
4月 03, 2023
作者:
C
congqixia
提交者:
GitHub
4月 03, 2023
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Use peek segment info in querynodev2 (#23183)
Signed-off-by:
N
Congqi Xia
<
congqi.xia@zilliz.com
>
上级
2a760c10
变更
9
隐藏空白更改
内联
并排
Showing
9 changed file
with
160 addition
and
55 deletion
+160
-55
internal/querynodev2/delegator/delegator.go
internal/querynodev2/delegator/delegator.go
+4
-3
internal/querynodev2/delegator/delegator_data_test.go
internal/querynodev2/delegator/delegator_data_test.go
+5
-9
internal/querynodev2/delegator/delegator_test.go
internal/querynodev2/delegator/delegator_test.go
+3
-5
internal/querynodev2/delegator/distribution.go
internal/querynodev2/delegator/distribution.go
+17
-6
internal/querynodev2/delegator/distribution_test.go
internal/querynodev2/delegator/distribution_test.go
+95
-0
internal/querynodev2/delegator/mock_delegator.go
internal/querynodev2/delegator/mock_delegator.go
+23
-14
internal/querynodev2/delegator/snapshot.go
internal/querynodev2/delegator/snapshot.go
+9
-9
internal/querynodev2/services.go
internal/querynodev2/services.go
+1
-3
internal/querynodev2/services_test.go
internal/querynodev2/services_test.go
+3
-6
未找到文件。
internal/querynodev2/delegator/delegator.go
浏览文件 @
e3e3ac15
...
...
@@ -75,7 +75,7 @@ func newLifetime() *lifetime {
type
ShardDelegator
interface
{
Collection
()
int64
Version
()
int64
Get
Distribution
()
*
distribution
Get
SegmentInfo
()
(
sealed
[]
SnapshotItem
,
growing
[]
SegmentEntry
)
SyncDistribution
(
ctx
context
.
Context
,
entries
...
SegmentEntry
)
Search
(
ctx
context
.
Context
,
req
*
querypb
.
SearchRequest
)
([]
*
internalpb
.
SearchResults
,
error
)
Query
(
ctx
context
.
Context
,
req
*
querypb
.
QueryRequest
)
([]
*
internalpb
.
RetrieveResults
,
error
)
...
...
@@ -161,8 +161,9 @@ func (sd *shardDelegator) Version() int64 {
return
sd
.
version
}
func
(
sd
*
shardDelegator
)
GetDistribution
()
*
distribution
{
return
sd
.
distribution
// GetSegmentInfo returns current segment distribution snapshot.
func
(
sd
*
shardDelegator
)
GetSegmentInfo
()
([]
SnapshotItem
,
[]
SegmentEntry
)
{
return
sd
.
distribution
.
Peek
()
}
// SyncDistribution revises distribution.
...
...
internal/querynodev2/delegator/delegator_data_test.go
浏览文件 @
e3e3ac15
...
...
@@ -319,7 +319,7 @@ func (s *DelegatorDataSuite) TestLoadSegments() {
})
s
.
NoError
(
err
)
sealed
,
_
,
_
:=
s
.
delegator
.
GetDistribution
()
.
GetCurrent
()
sealed
,
_
:=
s
.
delegator
.
GetSegmentInfo
()
s
.
Require
()
.
Equal
(
1
,
len
(
sealed
))
s
.
Equal
(
int64
(
1
),
sealed
[
0
]
.
NodeID
)
s
.
ElementsMatch
([]
SegmentEntry
{
...
...
@@ -395,7 +395,7 @@ func (s *DelegatorDataSuite) TestLoadSegments() {
})
s
.
NoError
(
err
)
sealed
,
_
,
_
:=
s
.
delegator
.
GetDistribution
()
.
GetCurrent
()
sealed
,
_
:=
s
.
delegator
.
GetSegmentInfo
()
s
.
Require
()
.
Equal
(
1
,
len
(
sealed
))
s
.
Equal
(
int64
(
1
),
sealed
[
0
]
.
NodeID
)
s
.
ElementsMatch
([]
SegmentEntry
{
...
...
@@ -410,7 +410,6 @@ func (s *DelegatorDataSuite) TestLoadSegments() {
PartitionID
:
500
,
},
},
sealed
[
0
]
.
Segments
)
})
s
.
Run
(
"get_worker_fail"
,
func
()
{
...
...
@@ -599,8 +598,7 @@ func (s *DelegatorDataSuite) TestReleaseSegment() {
})
s
.
Require
()
.
NoError
(
err
)
sealed
,
growing
,
version
:=
s
.
delegator
.
GetDistribution
()
.
GetCurrent
()
s
.
delegator
.
GetDistribution
()
.
FinishUsage
(
version
)
sealed
,
growing
:=
s
.
delegator
.
GetSegmentInfo
()
s
.
Require
()
.
Equal
(
1
,
len
(
sealed
))
s
.
Equal
(
int64
(
1
),
sealed
[
0
]
.
NodeID
)
s
.
ElementsMatch
([]
SegmentEntry
{
...
...
@@ -627,8 +625,7 @@ func (s *DelegatorDataSuite) TestReleaseSegment() {
},
false
)
s
.
NoError
(
err
)
sealed
,
_
,
version
=
s
.
delegator
.
GetDistribution
()
.
GetCurrent
()
s
.
delegator
.
GetDistribution
()
.
FinishUsage
(
version
)
sealed
,
_
=
s
.
delegator
.
GetSegmentInfo
()
s
.
Equal
(
0
,
len
(
sealed
))
err
=
s
.
delegator
.
ReleaseSegments
(
ctx
,
&
querypb
.
ReleaseSegmentsRequest
{
...
...
@@ -639,8 +636,7 @@ func (s *DelegatorDataSuite) TestReleaseSegment() {
},
false
)
s
.
NoError
(
err
)
_
,
growing
,
version
=
s
.
delegator
.
GetDistribution
()
.
GetCurrent
()
s
.
delegator
.
GetDistribution
()
.
FinishUsage
(
version
)
_
,
growing
=
s
.
delegator
.
GetSegmentInfo
()
s
.
Equal
(
0
,
len
(
growing
))
err
=
s
.
delegator
.
ReleaseSegments
(
ctx
,
&
querypb
.
ReleaseSegmentsRequest
{
...
...
internal/querynodev2/delegator/delegator_test.go
浏览文件 @
e3e3ac15
...
...
@@ -142,9 +142,8 @@ func (s *DelegatorSuite) TestBasicInfo() {
s
.
True
(
s
.
delegator
.
Serviceable
())
}
func
(
s
*
DelegatorSuite
)
TestDistribution
()
{
sealed
,
growing
,
version
:=
s
.
delegator
.
GetDistribution
()
.
GetCurrent
()
s
.
delegator
.
GetDistribution
()
.
FinishUsage
(
version
)
func
(
s
*
DelegatorSuite
)
TestGetSegmentInfo
()
{
sealed
,
growing
:=
s
.
delegator
.
GetSegmentInfo
()
s
.
Equal
(
0
,
len
(
sealed
))
s
.
Equal
(
0
,
len
(
growing
))
...
...
@@ -155,7 +154,7 @@ func (s *DelegatorSuite) TestDistribution() {
Version
:
2001
,
})
sealed
,
growing
,
version
=
s
.
delegator
.
GetDistribution
()
.
GetCurrent
()
sealed
,
growing
=
s
.
delegator
.
GetSegmentInfo
()
s
.
EqualValues
([]
SnapshotItem
{
{
NodeID
:
1
,
...
...
@@ -170,7 +169,6 @@ func (s *DelegatorSuite) TestDistribution() {
},
},
sealed
)
s
.
Equal
(
0
,
len
(
growing
))
s
.
delegator
.
GetDistribution
()
.
FinishUsage
(
version
)
}
func
(
s
*
DelegatorSuite
)
TestSearch
()
{
...
...
internal/querynodev2/delegator/distribution.go
浏览文件 @
e3e3ac15
...
...
@@ -45,7 +45,7 @@ type distribution struct {
snapshots
*
typeutil
.
ConcurrentMap
[
int64
,
*
snapshot
]
// current is the snapshot for quick usage for search/query
// generated for each change of distribution
current
*
snapshot
current
*
atomic
.
Pointer
[
snapshot
]
// protects current & segments
mut
sync
.
RWMutex
}
...
...
@@ -65,6 +65,7 @@ func NewDistribution() *distribution {
growingSegments
:
make
(
map
[
UniqueID
]
SegmentEntry
),
sealedSegments
:
make
(
map
[
UniqueID
]
SegmentEntry
),
snapshots
:
typeutil
.
NewConcurrentMap
[
int64
,
*
snapshot
](),
current
:
atomic
.
NewPointer
[
snapshot
](
nil
),
}
dist
.
genSnapshot
()
...
...
@@ -76,8 +77,9 @@ func (d *distribution) GetCurrent(partitions ...int64) (sealed []SnapshotItem, g
d
.
mut
.
RLock
()
defer
d
.
mut
.
RUnlock
()
sealed
,
growing
=
d
.
current
.
Get
(
partitions
...
)
version
=
d
.
current
.
version
current
:=
d
.
current
.
Load
()
sealed
,
growing
=
current
.
Get
(
partitions
...
)
version
=
current
.
version
return
}
...
...
@@ -89,6 +91,14 @@ func (d *distribution) FinishUsage(version int64) {
}
}
// Peek returns current snapshot without increasing inuse count
// show only used by GetDataDistribution.
func
(
d
*
distribution
)
Peek
(
partitions
...
int64
)
(
sealed
[]
SnapshotItem
,
growing
[]
SegmentEntry
)
{
current
:=
d
.
current
.
Load
()
sealed
,
growing
=
current
.
Peek
(
partitions
...
)
return
sealed
,
growing
}
// Serviceable returns wether current snapshot is serviceable.
func
(
d
*
distribution
)
Serviceable
()
bool
{
return
d
.
serviceable
.
Load
()
...
...
@@ -215,12 +225,13 @@ func (d *distribution) genSnapshot() chan struct{} {
// stores last snapshot
// ok to be nil
last
:=
d
.
current
last
:=
d
.
current
.
Load
()
// increase version
d
.
version
++
d
.
current
=
NewSnapshot
(
dist
,
growing
,
last
,
d
.
version
)
newSnapShot
:=
NewSnapshot
(
dist
,
growing
,
last
,
d
.
version
)
d
.
current
.
Store
(
newSnapShot
)
// shall be a new one
d
.
snapshots
.
GetOrInsert
(
d
.
version
,
d
.
curren
t
)
d
.
snapshots
.
GetOrInsert
(
d
.
version
,
newSnapSho
t
)
// first snapshot, return closed chan
if
last
==
nil
{
...
...
internal/querynodev2/delegator/distribution_test.go
浏览文件 @
e3e3ac15
...
...
@@ -390,6 +390,101 @@ func (s *DistributionSuite) TestRemoveDistribution() {
}
}
func
(
s
*
DistributionSuite
)
TestPeek
()
{
type
testCase
struct
{
tag
string
input
[]
SegmentEntry
expected
[]
SnapshotItem
}
cases
:=
[]
testCase
{
{
tag
:
"one node"
,
input
:
[]
SegmentEntry
{
{
NodeID
:
1
,
SegmentID
:
1
,
},
{
NodeID
:
1
,
SegmentID
:
2
,
},
},
expected
:
[]
SnapshotItem
{
{
NodeID
:
1
,
Segments
:
[]
SegmentEntry
{
{
NodeID
:
1
,
SegmentID
:
1
,
},
{
NodeID
:
1
,
SegmentID
:
2
,
},
},
},
},
},
{
tag
:
"multiple nodes"
,
input
:
[]
SegmentEntry
{
{
NodeID
:
1
,
SegmentID
:
1
,
},
{
NodeID
:
2
,
SegmentID
:
2
,
},
{
NodeID
:
1
,
SegmentID
:
3
,
},
},
expected
:
[]
SnapshotItem
{
{
NodeID
:
1
,
Segments
:
[]
SegmentEntry
{
{
NodeID
:
1
,
SegmentID
:
1
,
},
{
NodeID
:
1
,
SegmentID
:
3
,
},
},
},
{
NodeID
:
2
,
Segments
:
[]
SegmentEntry
{
{
NodeID
:
2
,
SegmentID
:
2
,
},
},
},
},
},
}
for
_
,
tc
:=
range
cases
{
s
.
Run
(
tc
.
tag
,
func
()
{
s
.
SetupTest
()
defer
s
.
TearDownTest
()
// peek during lock
s
.
dist
.
AddDistributions
(
tc
.
input
...
)
s
.
dist
.
mut
.
Lock
()
sealed
,
_
:=
s
.
dist
.
Peek
()
s
.
compareSnapshotItems
(
tc
.
expected
,
sealed
)
s
.
dist
.
mut
.
Unlock
()
})
}
}
func
TestDistributionSuite
(
t
*
testing
.
T
)
{
suite
.
Run
(
t
,
new
(
DistributionSuite
))
}
internal/querynodev2/delegator/mock_delegator.go
浏览文件 @
e3e3ac15
...
...
@@ -87,41 +87,50 @@ func (_c *MockShardDelegator_Collection_Call) Return(_a0 int64) *MockShardDelega
return
_c
}
// Get
Distribution
provides a mock function with given fields:
func
(
_m
*
MockShardDelegator
)
Get
Distribution
()
*
distribution
{
// Get
SegmentInfo
provides a mock function with given fields:
func
(
_m
*
MockShardDelegator
)
Get
SegmentInfo
()
([]
SnapshotItem
,
[]
SegmentEntry
)
{
ret
:=
_m
.
Called
()
var
r0
*
distribution
if
rf
,
ok
:=
ret
.
Get
(
0
)
.
(
func
()
*
distribution
);
ok
{
var
r0
[]
SnapshotItem
if
rf
,
ok
:=
ret
.
Get
(
0
)
.
(
func
()
[]
SnapshotItem
);
ok
{
r0
=
rf
()
}
else
{
if
ret
.
Get
(
0
)
!=
nil
{
r0
=
ret
.
Get
(
0
)
.
(
*
distribution
)
r0
=
ret
.
Get
(
0
)
.
(
[]
SnapshotItem
)
}
}
return
r0
var
r1
[]
SegmentEntry
if
rf
,
ok
:=
ret
.
Get
(
1
)
.
(
func
()
[]
SegmentEntry
);
ok
{
r1
=
rf
()
}
else
{
if
ret
.
Get
(
1
)
!=
nil
{
r1
=
ret
.
Get
(
1
)
.
([]
SegmentEntry
)
}
}
return
r0
,
r1
}
// MockShardDelegator_Get
Distribution_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetDistribution
'
type
MockShardDelegator_Get
Distribution
_Call
struct
{
// MockShardDelegator_Get
SegmentInfo_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetSegmentInfo
'
type
MockShardDelegator_Get
SegmentInfo
_Call
struct
{
*
mock
.
Call
}
// Get
Distribution
is a helper method to define mock.On call
func
(
_e
*
MockShardDelegator_Expecter
)
Get
Distribution
()
*
MockShardDelegator_GetDistribution
_Call
{
return
&
MockShardDelegator_Get
Distribution_Call
{
Call
:
_e
.
mock
.
On
(
"GetDistribution
"
)}
// Get
SegmentInfo
is a helper method to define mock.On call
func
(
_e
*
MockShardDelegator_Expecter
)
Get
SegmentInfo
()
*
MockShardDelegator_GetSegmentInfo
_Call
{
return
&
MockShardDelegator_Get
SegmentInfo_Call
{
Call
:
_e
.
mock
.
On
(
"GetSegmentInfo
"
)}
}
func
(
_c
*
MockShardDelegator_Get
Distribution_Call
)
Run
(
run
func
())
*
MockShardDelegator_GetDistribution
_Call
{
func
(
_c
*
MockShardDelegator_Get
SegmentInfo_Call
)
Run
(
run
func
())
*
MockShardDelegator_GetSegmentInfo
_Call
{
_c
.
Call
.
Run
(
func
(
args
mock
.
Arguments
)
{
run
()
})
return
_c
}
func
(
_c
*
MockShardDelegator_Get
Distribution_Call
)
Return
(
_a0
*
distribution
)
*
MockShardDelegator_GetDistribution
_Call
{
_c
.
Call
.
Return
(
_a0
)
func
(
_c
*
MockShardDelegator_Get
SegmentInfo_Call
)
Return
(
sealed
[]
SnapshotItem
,
growing
[]
SegmentEntry
)
*
MockShardDelegator_GetSegmentInfo
_Call
{
_c
.
Call
.
Return
(
sealed
,
growing
)
return
_c
}
...
...
internal/querynodev2/delegator/snapshot.go
浏览文件 @
e3e3ac15
...
...
@@ -76,6 +76,15 @@ func (s *snapshot) Expire(cleanup snapshotCleanup) {
func
(
s
*
snapshot
)
Get
(
partitions
...
int64
)
(
sealed
[]
SnapshotItem
,
growing
[]
SegmentEntry
)
{
s
.
inUse
.
Inc
()
return
s
.
filter
(
partitions
...
)
}
// Peek returns segment distributions without increasing inUse.
func
(
s
*
snapshot
)
Peek
(
partitions
...
int64
)
(
sealed
[]
SnapshotItem
,
growing
[]
SegmentEntry
)
{
return
s
.
filter
(
partitions
...
)
}
func
(
s
*
snapshot
)
filter
(
partitions
...
int64
)
(
sealed
[]
SnapshotItem
,
growing
[]
SegmentEntry
)
{
filter
:=
func
(
entry
SegmentEntry
,
idx
int
)
bool
{
return
len
(
partitions
)
==
0
||
funcutil
.
SliceContain
(
partitions
,
entry
.
PartitionID
)
}
...
...
@@ -121,12 +130,3 @@ func (s *snapshot) checkCleared(cleanup snapshotCleanup) {
})
}
}
func
inList
(
list
[]
int64
,
target
int64
)
bool
{
for
_
,
i
:=
range
list
{
if
i
==
target
{
return
true
}
}
return
false
}
internal/querynodev2/services.go
浏览文件 @
e3e3ac15
...
...
@@ -902,9 +902,7 @@ func (node *QueryNode) GetDataDistribution(ctx context.Context, req *querypb.Get
Version
:
value
.
Version
(),
})
sealed
,
growing
,
version
:=
value
.
GetDistribution
()
.
GetCurrent
()
value
.
GetDistribution
()
.
FinishUsage
(
version
)
sealed
,
growing
:=
value
.
GetSegmentInfo
()
sealedSegments
:=
make
(
map
[
int64
]
*
querypb
.
SegmentDist
)
for
_
,
item
:=
range
sealed
{
for
_
,
segment
:=
range
item
.
Segments
{
...
...
internal/querynodev2/services_test.go
浏览文件 @
e3e3ac15
...
...
@@ -1190,9 +1190,8 @@ func (suite *ServiceSuite) TestSyncDistribution_ReleaseResultCheck() {
delegator
,
ok
:=
suite
.
node
.
delegators
.
Get
(
suite
.
vchannel
)
suite
.
True
(
ok
)
sealedSegments
,
_
,
version
:=
delegator
.
GetDistribution
()
.
GetCurrent
()
sealedSegments
,
_
:=
delegator
.
GetSegmentInfo
()
suite
.
Len
(
sealedSegments
[
0
]
.
Segments
,
3
)
delegator
.
GetDistribution
()
.
FinishUsage
(
version
)
// data
req
:=
&
querypb
.
SyncDistributionRequest
{
...
...
@@ -1215,9 +1214,8 @@ func (suite *ServiceSuite) TestSyncDistribution_ReleaseResultCheck() {
status
,
err
:=
suite
.
node
.
SyncDistribution
(
ctx
,
req
)
suite
.
NoError
(
err
)
suite
.
Equal
(
commonpb
.
ErrorCode_Success
,
status
.
ErrorCode
)
sealedSegments
,
_
,
version
=
delegator
.
GetDistribution
()
.
GetCurrent
()
sealedSegments
,
_
=
delegator
.
GetSegmentInfo
()
suite
.
Len
(
sealedSegments
[
0
]
.
Segments
,
3
)
delegator
.
GetDistribution
()
.
FinishUsage
(
version
)
releaseAction
=
&
querypb
.
SyncAction
{
Type
:
querypb
.
SyncType_Remove
,
...
...
@@ -1230,9 +1228,8 @@ func (suite *ServiceSuite) TestSyncDistribution_ReleaseResultCheck() {
status
,
err
=
suite
.
node
.
SyncDistribution
(
ctx
,
req
)
suite
.
NoError
(
err
)
suite
.
Equal
(
commonpb
.
ErrorCode_Success
,
status
.
ErrorCode
)
sealedSegments
,
_
,
version
=
delegator
.
GetDistribution
()
.
GetCurrent
()
sealedSegments
,
_
=
delegator
.
GetSegmentInfo
()
suite
.
Len
(
sealedSegments
[
0
]
.
Segments
,
2
)
delegator
.
GetDistribution
()
.
FinishUsage
(
version
)
}
func
(
suite
*
ServiceSuite
)
TestSyncDistribution_Failed
()
{
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录