Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
f31ed089
M
milvus
项目概览
milvus
/
milvus
11 个月 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
f31ed089
编写于
11月 26, 2021
作者:
G
godchen
提交者:
GitHub
11月 26, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Add timeout for reader next (#12308)
Signed-off-by:
N
godchen
<
qingxiang.chen@zilliz.com
>
上级
e9be4a81
变更
6
隐藏空白更改
内联
并排
Showing
6 changed file
with
57 addition
and
26 deletion
+57
-26
internal/datanode/flow_graph_dmstream_input_node_test.go
internal/datanode/flow_graph_dmstream_input_node_test.go
+3
-0
internal/msgstream/mq_msgstream.go
internal/msgstream/mq_msgstream.go
+22
-20
internal/msgstream/mq_msgstream_test.go
internal/msgstream/mq_msgstream_test.go
+16
-4
internal/msgstream/msgstream.go
internal/msgstream/msgstream.go
+1
-0
internal/proxy/mock_test.go
internal/proxy/mock_test.go
+4
-0
internal/querynode/segment_loader.go
internal/querynode/segment_loader.go
+11
-2
未找到文件。
internal/datanode/flow_graph_dmstream_input_node_test.go
浏览文件 @
f31ed089
...
...
@@ -104,6 +104,9 @@ func (mtm *mockTtMsgStream) SeekReaders(msgPositions []*internalpb.MsgPosition)
func
(
mtm
*
mockTtMsgStream
)
Next
(
ctx
context
.
Context
,
channelName
string
)
(
msgstream
.
TsMsg
,
error
)
{
return
nil
,
nil
}
func
(
mtm
*
mockTtMsgStream
)
HasNext
(
channelName
string
)
bool
{
return
true
}
func
TestNewDmInputNode
(
t
*
testing
.
T
)
{
ctx
:=
context
.
Background
()
...
...
internal/msgstream/mq_msgstream.go
浏览文件 @
f31ed089
...
...
@@ -575,27 +575,29 @@ func (ms *mqMsgStream) Next(ctx context.Context, channelName string) (TsMsg, err
if
!
ok
{
return
nil
,
fmt
.
Errorf
(
"reader for channel %s is not exist"
,
channelName
)
}
if
reader
.
HasNext
()
{
msg
,
err
:=
reader
.
Next
(
ctx
)
if
err
!=
nil
{
return
nil
,
err
}
tsMsg
,
err
:=
ms
.
getTsMsgFromConsumerMsg
(
msg
)
if
err
!=
nil
{
log
.
Error
(
"Failed to getTsMsgFromConsumerMsg"
,
zap
.
Error
(
err
))
return
nil
,
errors
.
New
(
"Failed to getTsMsgFromConsumerMsg"
)
}
pos
:=
tsMsg
.
Position
()
tsMsg
.
SetPosition
(
&
MsgPosition
{
ChannelName
:
pos
.
ChannelName
,
MsgID
:
pos
.
MsgID
,
Timestamp
:
tsMsg
.
BeginTs
(),
})
return
tsMsg
,
nil
msg
,
err
:=
reader
.
Next
(
ctx
)
if
err
!=
nil
{
return
nil
,
err
}
log
.
Debug
(
"All data has been read, there is no more data"
,
zap
.
String
(
"channel"
,
channelName
))
return
nil
,
nil
tsMsg
,
err
:=
ms
.
getTsMsgFromConsumerMsg
(
msg
)
if
err
!=
nil
{
log
.
Error
(
"Failed to getTsMsgFromConsumerMsg"
,
zap
.
Error
(
err
))
return
nil
,
errors
.
New
(
"Failed to getTsMsgFromConsumerMsg"
)
}
pos
:=
tsMsg
.
Position
()
tsMsg
.
SetPosition
(
&
MsgPosition
{
ChannelName
:
pos
.
ChannelName
,
MsgID
:
pos
.
MsgID
,
Timestamp
:
tsMsg
.
BeginTs
(),
})
return
tsMsg
,
nil
}
func
(
ms
*
mqMsgStream
)
HasNext
(
channelName
string
)
bool
{
reader
,
ok
:=
ms
.
readers
[
channelName
]
if
!
ok
{
return
false
}
return
reader
.
HasNext
()
}
// Seek reset the subscription associated with this consumer to a specific position, the seek position is exclusive
...
...
internal/msgstream/mq_msgstream_test.go
浏览文件 @
f31ed089
...
...
@@ -1296,6 +1296,8 @@ func TestStream_MqMsgStream_Reader(t *testing.T) {
defer
readStream
.
Close
()
var
seekPosition
*
internalpb
.
MsgPosition
for
i
:=
0
;
i
<
n
;
i
++
{
hasNext
:=
readStream
.
HasNext
(
c
)
assert
.
True
(
t
,
hasNext
)
result
,
err
:=
readStream
.
Next
(
ctx
,
c
)
assert
.
Nil
(
t
,
err
)
assert
.
Equal
(
t
,
result
.
ID
(),
int64
(
i
))
...
...
@@ -1303,8 +1305,12 @@ func TestStream_MqMsgStream_Reader(t *testing.T) {
seekPosition
=
result
.
Position
()
}
}
result
,
err
:=
readStream
.
Next
(
ctx
,
c
)
assert
.
Nil
(
t
,
err
)
hasNext
:=
readStream
.
HasNext
(
c
)
assert
.
False
(
t
,
hasNext
)
timeoutCtx1
,
cancel
:=
context
.
WithTimeout
(
ctx
,
3
*
time
.
Second
)
defer
cancel
()
result
,
err
:=
readStream
.
Next
(
timeoutCtx1
,
c
)
assert
.
NotNil
(
t
,
err
)
assert
.
Nil
(
t
,
result
)
readStream2
:=
getPulsarReader
(
pulsarAddress
,
readerChannels
)
...
...
@@ -1312,12 +1318,18 @@ func TestStream_MqMsgStream_Reader(t *testing.T) {
readStream2
.
SeekReaders
([]
*
internalpb
.
MsgPosition
{
seekPosition
})
for
i
:=
p
;
i
<
10
;
i
++
{
hasNext
:=
readStream2
.
HasNext
(
c
)
assert
.
True
(
t
,
hasNext
)
result
,
err
:=
readStream2
.
Next
(
ctx
,
c
)
assert
.
Nil
(
t
,
err
)
assert
.
Equal
(
t
,
result
.
ID
(),
int64
(
i
))
}
result2
,
err
:=
readStream2
.
Next
(
ctx
,
c
)
assert
.
Nil
(
t
,
err
)
hasNext
=
readStream2
.
HasNext
(
c
)
assert
.
False
(
t
,
hasNext
)
timeoutCtx2
,
cancel
:=
context
.
WithTimeout
(
ctx
,
3
*
time
.
Second
)
defer
cancel
()
result2
,
err
:=
readStream2
.
Next
(
timeoutCtx2
,
c
)
assert
.
NotNil
(
t
,
err
)
assert
.
Nil
(
t
,
result2
)
}
...
...
internal/msgstream/msgstream.go
浏览文件 @
f31ed089
...
...
@@ -69,6 +69,7 @@ type MsgStream interface {
BroadcastMark
(
*
MsgPack
)
(
map
[
string
][]
MessageID
,
error
)
Consume
()
*
MsgPack
Next
(
ctx
context
.
Context
,
channelName
string
)
(
TsMsg
,
error
)
HasNext
(
channelName
string
)
bool
Seek
(
offset
[]
*
MsgPosition
)
error
SeekReaders
(
msgPositions
[]
*
internalpb
.
MsgPosition
)
error
}
...
...
internal/proxy/mock_test.go
浏览文件 @
f31ed089
...
...
@@ -288,6 +288,10 @@ func (ms *simpleMockMsgStream) Next(ctx context.Context, channelName string) (ms
return
nil
,
nil
}
func
(
ms
*
simpleMockMsgStream
)
HasNext
(
channelName
string
)
bool
{
return
true
}
func
(
ms
*
simpleMockMsgStream
)
AsConsumerWithPosition
(
channels
[]
string
,
subName
string
,
position
mqclient
.
SubscriptionInitialPosition
)
{
}
...
...
internal/querynode/segment_loader.go
浏览文件 @
f31ed089
...
...
@@ -18,6 +18,7 @@ import (
"path"
"strconv"
"sync"
"time"
"go.uber.org/zap"
...
...
@@ -37,6 +38,8 @@ import (
"github.com/milvus-io/milvus/internal/util/funcutil"
)
const
timeoutForEachRead
=
10
*
time
.
Second
// segmentLoader is only responsible for loading the field data from binlog
type
segmentLoader
struct
{
historicalReplica
ReplicaInterface
...
...
@@ -458,24 +461,30 @@ func (loader *segmentLoader) FromDmlCPLoadDelete(ctx context.Context, collection
deleteOffset
:
make
(
map
[
UniqueID
]
int64
),
}
log
.
Debug
(
"start read msg from stream reader"
)
for
{
for
stream
.
HasNext
(
pChannelName
)
{
ctx
,
cancel
:=
context
.
WithTimeout
(
ctx
,
timeoutForEachRead
)
tsMsg
,
err
:=
stream
.
Next
(
ctx
,
pChannelName
)
if
err
!=
nil
{
cancel
()
return
err
}
if
tsMsg
==
nil
{
break
cancel
()
continue
}
if
tsMsg
.
Type
()
==
commonpb
.
MsgType_Delete
{
dmsg
:=
tsMsg
.
(
*
msgstream
.
DeleteMsg
)
if
dmsg
.
CollectionID
!=
collectionID
{
cancel
()
continue
}
log
.
Debug
(
"delete pk"
,
zap
.
Any
(
"pk"
,
dmsg
.
PrimaryKeys
))
processDeleteMessages
(
loader
.
historicalReplica
,
dmsg
,
delData
)
}
cancel
()
}
log
.
Debug
(
"All data has been read, there is no more data"
,
zap
.
String
(
"channel"
,
pChannelName
))
for
segmentID
,
pks
:=
range
delData
.
deleteIDs
{
segment
,
err
:=
loader
.
historicalReplica
.
getSegmentByID
(
segmentID
)
if
err
!=
nil
{
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录