Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
921ef0f2
M
milvus
项目概览
milvus
/
milvus
11 个月 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
921ef0f2
编写于
10月 27, 2021
作者:
B
bigsheeper
提交者:
GitHub
10月 27, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Fix seek query channel error (#10723)
Signed-off-by:
N
bigsheeper
<
yihao.dai@zilliz.com
>
上级
87042932
变更
3
隐藏空白更改
内联
并排
Showing
3 changed file
with
67 addition
and
9 deletion
+67
-9
Makefile
Makefile
+5
-0
internal/querycoord/meta.go
internal/querycoord/meta.go
+36
-5
internal/querycoord/meta_test.go
internal/querycoord/meta_test.go
+26
-4
未找到文件。
Makefile
浏览文件 @
921ef0f2
...
...
@@ -116,6 +116,11 @@ test-proxy:
go
test
-race
-coverpkg
=
./...
-coverprofile
=
profile.out
-covermode
=
atomic
-timeout
5m github.com/milvus-io/milvus/internal/proxy
-v
test-querycoord
:
@
echo
"Running go unittests..."
go
test
-race
-coverpkg
=
./...
-coverprofile
=
profile.out
-covermode
=
atomic
-timeout
5m github.com/milvus-io/milvus/internal/querycoord
-v
test-go
:
build-cpp-with-unittest
@
echo
"Running go unittests..."
@
(
env
bash
$(PWD)
/scripts/run_go_codecov.sh
)
...
...
internal/querycoord/meta.go
浏览文件 @
921ef0f2
...
...
@@ -37,6 +37,7 @@ const (
segmentMetaPrefix
=
"queryCoord-segmentMeta"
queryChannelMetaPrefix
=
"queryCoord-queryChannel"
sealedSegmentChangeInfoPrefix
=
"queryCoord-sealedSegmentChangeInfo"
globalQuerySeekPositionPrefix
=
"queryCoord-globalQuerySeekPosition"
)
type
col2SegmentInfos
=
map
[
UniqueID
][]
*
querypb
.
SegmentInfo
...
...
@@ -100,6 +101,7 @@ type MetaReplica struct {
queryStreams
map
[
UniqueID
]
msgstream
.
MsgStream
streamMu
sync
.
RWMutex
globalSeekPosition
*
internalpb
.
MsgPosition
//partitionStates map[UniqueID]*querypb.PartitionStates
}
...
...
@@ -109,6 +111,7 @@ func newMeta(ctx context.Context, kv kv.MetaKv, factory msgstream.Factory, idAll
segmentInfos
:=
make
(
map
[
UniqueID
]
*
querypb
.
SegmentInfo
)
queryChannelInfos
:=
make
(
map
[
UniqueID
]
*
querypb
.
QueryChannelInfo
)
queryMsgStream
:=
make
(
map
[
UniqueID
]
msgstream
.
MsgStream
)
position
:=
&
internalpb
.
MsgPosition
{}
m
:=
&
MetaReplica
{
ctx
:
childCtx
,
...
...
@@ -117,10 +120,11 @@ func newMeta(ctx context.Context, kv kv.MetaKv, factory msgstream.Factory, idAll
msFactory
:
factory
,
idAllocator
:
idAllocator
,
collectionInfos
:
collectionInfos
,
segmentInfos
:
segmentInfos
,
queryChannelInfos
:
queryChannelInfos
,
queryStreams
:
queryMsgStream
,
collectionInfos
:
collectionInfos
,
segmentInfos
:
segmentInfos
,
queryChannelInfos
:
queryChannelInfos
,
queryStreams
:
queryMsgStream
,
globalSeekPosition
:
position
,
}
err
:=
m
.
reloadFromKV
()
...
...
@@ -182,6 +186,15 @@ func (m *MetaReplica) reloadFromKV() error {
}
m
.
queryChannelInfos
[
collectionID
]
=
queryChannelInfo
}
globalSeekPosValue
,
err
:=
m
.
client
.
Load
(
globalQuerySeekPositionPrefix
)
if
err
==
nil
{
position
:=
&
internalpb
.
MsgPosition
{}
err
=
proto
.
Unmarshal
([]
byte
(
globalSeekPosValue
),
position
)
if
err
!=
nil
{
return
err
}
m
.
globalSeekPosition
=
position
}
//TODO::update partition states
return
nil
...
...
@@ -399,6 +412,7 @@ func (m *MetaReplica) saveGlobalSealedSegInfos(saves col2SegmentInfos) (col2Seal
}
queryChannelInfosMap
:=
make
(
map
[
UniqueID
]
*
querypb
.
QueryChannelInfo
)
var
globalSeekPositionTmp
*
internalpb
.
MsgPosition
for
collectionID
,
segmentChangeInfos
:=
range
col2SegmentChangeInfos
{
// get msgStream to produce sealedSegmentChangeInfos to query channel
queryChannelInfo
,
messageIDInfos
,
err
:=
m
.
sendSealedSegmentChangeInfos
(
collectionID
,
segmentChangeInfos
)
...
...
@@ -438,6 +452,7 @@ func (m *MetaReplica) saveGlobalSealedSegInfos(saves col2SegmentInfos) (col2Seal
}
queryChannelInfo
.
GlobalSealedSegments
=
globalSealedSegmentInfos
queryChannelInfosMap
[
collectionID
]
=
queryChannelInfo
globalSeekPositionTmp
=
queryChannelInfo
.
SeekPosition
}
saveKvs
:=
make
(
map
[
string
]
string
)
...
...
@@ -460,6 +475,11 @@ func (m *MetaReplica) saveGlobalSealedSegInfos(saves col2SegmentInfos) (col2Seal
channelKey
:=
fmt
.
Sprintf
(
"%s/%d"
,
queryChannelMetaPrefix
,
collectionID
)
saveKvs
[
channelKey
]
=
string
(
channelInfoBytes
)
}
seekPos
,
err
:=
proto
.
Marshal
(
globalSeekPositionTmp
)
if
err
!=
nil
{
return
col2SegmentChangeInfos
,
err
}
saveKvs
[
globalQuerySeekPositionPrefix
]
=
string
(
seekPos
)
// save segmentChangeInfo into etcd, query node will deal the changeInfo if the msgID key exist in etcd
// avoid the produce process success but save meta to etcd failed
...
...
@@ -474,7 +494,7 @@ func (m *MetaReplica) saveGlobalSealedSegInfos(saves col2SegmentInfos) (col2Seal
saveKvs
[
changeInfoKey
]
=
string
(
changeInfoBytes
)
}
err
:
=
m
.
client
.
MultiSave
(
saveKvs
)
err
=
m
.
client
.
MultiSave
(
saveKvs
)
if
err
!=
nil
{
log
.
Error
(
"updateGlobalSealedSegmentInfos: save info to etcd error"
,
zap
.
Error
(
err
))
return
col2SegmentChangeInfos
,
err
...
...
@@ -492,6 +512,7 @@ func (m *MetaReplica) saveGlobalSealedSegInfos(saves col2SegmentInfos) (col2Seal
for
collectionID
,
channelInfo
:=
range
queryChannelInfosMap
{
m
.
queryChannelInfos
[
collectionID
]
=
channelInfo
}
m
.
globalSeekPosition
=
globalSeekPositionTmp
m
.
channelMu
.
Unlock
()
return
col2SegmentChangeInfos
,
nil
...
...
@@ -563,6 +584,11 @@ func (m *MetaReplica) removeGlobalSealedSegInfos(collectionID UniqueID, partitio
}
channelKey
:=
fmt
.
Sprintf
(
"%s/%d"
,
queryChannelMetaPrefix
,
collectionID
)
saveKvs
[
channelKey
]
=
string
(
channelInfoBytes
)
seekPos
,
err
:=
proto
.
Marshal
(
queryChannelInfo
.
SeekPosition
)
if
err
!=
nil
{
return
col2SealedSegmentChangeInfos
{
collectionID
:
segmentChangeInfos
},
err
}
saveKvs
[
globalQuerySeekPositionPrefix
]
=
string
(
seekPos
)
// save segmentChangeInfo into etcd, query node will deal the changeInfo if the msgID key exist in etcd
// avoid the produce process success but save meta to etcd failed
...
...
@@ -594,6 +620,7 @@ func (m *MetaReplica) removeGlobalSealedSegInfos(collectionID UniqueID, partitio
m
.
channelMu
.
Lock
()
m
.
queryChannelInfos
[
collectionID
]
=
queryChannelInfo
m
.
globalSeekPosition
=
queryChannelInfo
.
SeekPosition
m
.
channelMu
.
Unlock
()
return
col2SealedSegmentChangeInfos
{
collectionID
:
segmentChangeInfos
},
nil
...
...
@@ -890,6 +917,10 @@ func (m *MetaReplica) getQueryChannelInfoByID(collectionID UniqueID) (*querypb.Q
// set info.collectionID from 0 to realID
info
.
CollectionID
=
collectionID
m
.
queryChannelInfos
[
collectionID
]
=
info
info
.
SeekPosition
=
m
.
globalSeekPosition
if
info
.
SeekPosition
!=
nil
{
info
.
SeekPosition
.
ChannelName
=
info
.
QueryChannelID
}
return
proto
.
Clone
(
info
)
.
(
*
querypb
.
QueryChannelInfo
),
nil
}
...
...
internal/querycoord/meta_test.go
浏览文件 @
921ef0f2
...
...
@@ -23,6 +23,7 @@ import (
"github.com/milvus-io/milvus/internal/kv"
etcdkv
"github.com/milvus-io/milvus/internal/kv/etcd"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/proto/querypb"
)
...
...
@@ -50,6 +51,10 @@ func (tk *testKv) LoadWithPrefix(key string) ([]string, []string, error) {
return
nil
,
nil
,
nil
}
func
(
tk
*
testKv
)
Load
(
key
string
)
(
string
,
error
)
{
return
""
,
nil
}
func
TestReplica_Release
(
t
*
testing
.
T
)
{
refreshParams
()
etcdKV
,
err
:=
etcdkv
.
NewEtcdKV
(
Params
.
EtcdEndpoints
,
Params
.
MetaRootPath
)
...
...
@@ -85,10 +90,11 @@ func TestMetaFunc(t *testing.T) {
kv
,
err
:=
etcdkv
.
NewEtcdKV
(
Params
.
EtcdEndpoints
,
Params
.
MetaRootPath
)
assert
.
Nil
(
t
,
err
)
meta
:=
&
MetaReplica
{
client
:
kv
,
collectionInfos
:
map
[
UniqueID
]
*
querypb
.
CollectionInfo
{},
segmentInfos
:
map
[
UniqueID
]
*
querypb
.
SegmentInfo
{},
queryChannelInfos
:
map
[
UniqueID
]
*
querypb
.
QueryChannelInfo
{},
client
:
kv
,
collectionInfos
:
map
[
UniqueID
]
*
querypb
.
CollectionInfo
{},
segmentInfos
:
map
[
UniqueID
]
*
querypb
.
SegmentInfo
{},
queryChannelInfos
:
map
[
UniqueID
]
*
querypb
.
QueryChannelInfo
{},
globalSeekPosition
:
&
internalpb
.
MsgPosition
{},
}
nodeID
:=
int64
(
100
)
...
...
@@ -342,4 +348,20 @@ func TestReloadMetaFromKV(t *testing.T) {
assert
.
Equal
(
t
,
true
,
ok
)
_
,
ok
=
meta
.
queryChannelInfos
[
defaultCollectionID
]
assert
.
Equal
(
t
,
true
,
ok
)
t
.
Run
(
"test no global query seek position"
,
func
(
t
*
testing
.
T
)
{
err
=
kv
.
Remove
(
globalQuerySeekPositionPrefix
)
assert
.
NoError
(
t
,
err
)
err
=
meta
.
reloadFromKV
()
assert
.
NoError
(
t
,
err
)
})
t
.
Run
(
"test wrong global query seek position"
,
func
(
t
*
testing
.
T
)
{
err
=
kv
.
Save
(
globalQuerySeekPositionPrefix
,
"&%*&^*^(&%*&%&^%"
)
assert
.
NoError
(
t
,
err
)
err
=
meta
.
reloadFromKV
()
assert
.
Error
(
t
,
err
)
})
}
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录