Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
0d499a9f
M
milvus
项目概览
milvus
/
milvus
11 个月 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
0d499a9f
编写于
3月 09, 2021
作者:
Y
yukun
提交者:
yefu.chen
3月 09, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Fix search waiting for seconds in singlenode
Signed-off-by:
N
yukun
<
kun.yu@zilliz.com
>
上级
408fe6e7
变更
3
显示空白变更内容
内联
并排
Showing
3 changed file
with
44 addition
and
32 deletion
+44
-32
internal/msgstream/rmqms/rmq_msgstream.go
internal/msgstream/rmqms/rmq_msgstream.go
+40
-29
internal/util/rocksmq/global_rmq.go
internal/util/rocksmq/global_rmq.go
+1
-1
internal/util/rocksmq/rocksmq.go
internal/util/rocksmq/rocksmq.go
+3
-2
未找到文件。
internal/msgstream/rmqms/rmq_msgstream.go
浏览文件 @
0d499a9f
...
@@ -84,7 +84,7 @@ func (ms *RmqMsgStream) Close() {
...
@@ -84,7 +84,7 @@ func (ms *RmqMsgStream) Close() {
}
}
for
_
,
consumer
:=
range
ms
.
consumers
{
for
_
,
consumer
:=
range
ms
.
consumers
{
_
=
rocksmq
.
Rmq
.
DestroyConsumerGroup
(
consumer
.
GroupName
,
consumer
.
ChannelName
)
_
=
rocksmq
.
Rmq
.
DestroyConsumerGroup
(
consumer
.
GroupName
,
consumer
.
ChannelName
)
close
(
consumer
.
Msg
Num
)
close
(
consumer
.
Msg
Mutex
)
}
}
}
}
...
@@ -112,12 +112,13 @@ func (ms *RmqMsgStream) AsConsumer(channels []string, groupName string) {
...
@@ -112,12 +112,13 @@ func (ms *RmqMsgStream) AsConsumer(channels []string, groupName string) {
for
_
,
channelName
:=
range
channels
{
for
_
,
channelName
:=
range
channels
{
consumer
,
err
:=
rocksmq
.
Rmq
.
CreateConsumerGroup
(
groupName
,
channelName
)
consumer
,
err
:=
rocksmq
.
Rmq
.
CreateConsumerGroup
(
groupName
,
channelName
)
if
err
==
nil
{
if
err
==
nil
{
consumer
.
MsgNum
=
make
(
chan
int
,
ms
.
rmqBufSize
)
consumer
.
MsgMutex
=
make
(
chan
struct
{},
ms
.
rmqBufSize
)
//consumer.MsgMutex <- struct{}{}
ms
.
consumers
=
append
(
ms
.
consumers
,
*
consumer
)
ms
.
consumers
=
append
(
ms
.
consumers
,
*
consumer
)
ms
.
consumerChannels
=
append
(
ms
.
consumerChannels
,
channelName
)
ms
.
consumerChannels
=
append
(
ms
.
consumerChannels
,
channelName
)
ms
.
consumerReflects
=
append
(
ms
.
consumerReflects
,
reflect
.
SelectCase
{
ms
.
consumerReflects
=
append
(
ms
.
consumerReflects
,
reflect
.
SelectCase
{
Dir
:
reflect
.
SelectRecv
,
Dir
:
reflect
.
SelectRecv
,
Chan
:
reflect
.
ValueOf
(
consumer
.
Msg
Num
),
Chan
:
reflect
.
ValueOf
(
consumer
.
Msg
Mutex
),
})
})
ms
.
wait
.
Add
(
1
)
ms
.
wait
.
Add
(
1
)
go
ms
.
receiveMsg
(
*
consumer
)
go
ms
.
receiveMsg
(
*
consumer
)
...
@@ -244,30 +245,35 @@ func (ms *RmqMsgStream) receiveMsg(consumer rocksmq.Consumer) {
...
@@ -244,30 +245,35 @@ func (ms *RmqMsgStream) receiveMsg(consumer rocksmq.Consumer) {
select
{
select
{
case
<-
ms
.
ctx
.
Done
()
:
case
<-
ms
.
ctx
.
Done
()
:
return
return
case
msgNum
,
ok
:=
<-
consumer
.
MsgNum
:
case
_
,
ok
:=
<-
consumer
.
MsgMutex
:
if
!
ok
{
if
!
ok
{
return
return
}
}
rmqMsg
,
err
:=
rocksmq
.
Rmq
.
Consume
(
consumer
.
GroupName
,
consumer
.
ChannelName
,
msgNum
)
tsMsgList
:=
make
([]
msgstream
.
TsMsg
,
0
)
for
{
rmqMsgs
,
err
:=
rocksmq
.
Rmq
.
Consume
(
consumer
.
GroupName
,
consumer
.
ChannelName
,
1
)
if
err
!=
nil
{
if
err
!=
nil
{
log
.
Printf
(
"Failed to consume message in rocksmq, error = %v"
,
err
)
log
.
Printf
(
"Failed to consume message in rocksmq, error = %v"
,
err
)
continue
continue
}
}
tsMsgList
:=
make
([]
msgstream
.
TsMsg
,
0
)
if
len
(
rmqMsgs
)
==
0
{
for
j
:=
0
;
j
<
len
(
rmqMsg
);
j
++
{
break
}
rmqMsg
:=
rmqMsgs
[
0
]
headerMsg
:=
commonpb
.
MsgHeader
{}
headerMsg
:=
commonpb
.
MsgHeader
{}
err
:=
proto
.
Unmarshal
(
rmqMsg
[
j
]
.
Payload
,
&
headerMsg
)
err
=
proto
.
Unmarshal
(
rmqMsg
.
Payload
,
&
headerMsg
)
if
err
!=
nil
{
if
err
!=
nil
{
log
.
Printf
(
"Failed to unmarshal message header, error = %v"
,
err
)
log
.
Printf
(
"Failed to unmar
`
shal message header, error = %v"
,
err
)
continue
continue
}
}
tsMsg
,
err
:=
ms
.
unmarshal
.
Unmarshal
(
rmqMsg
[
j
]
.
Payload
,
headerMsg
.
Base
.
MsgType
)
tsMsg
,
err
:=
ms
.
unmarshal
.
Unmarshal
(
rmqMsg
.
Payload
,
headerMsg
.
Base
.
MsgType
)
if
err
!=
nil
{
if
err
!=
nil
{
log
.
Printf
(
"Failed to unmarshal tsMsg, error = %v"
,
err
)
log
.
Printf
(
"Failed to unmarshal tsMsg, error = %v"
,
err
)
continue
continue
}
}
tsMsgList
=
append
(
tsMsgList
,
tsMsg
)
tsMsgList
=
append
(
tsMsgList
,
tsMsg
)
}
}
if
len
(
tsMsgList
)
>
0
{
if
len
(
tsMsgList
)
>
0
{
msgPack
:=
util
.
MsgPack
{
Msgs
:
tsMsgList
}
msgPack
:=
util
.
MsgPack
{
Msgs
:
tsMsgList
}
ms
.
receiveBuf
<-
&
msgPack
ms
.
receiveBuf
<-
&
msgPack
...
@@ -326,12 +332,13 @@ func (ms *RmqTtMsgStream) AsConsumer(channels []string,
...
@@ -326,12 +332,13 @@ func (ms *RmqTtMsgStream) AsConsumer(channels []string,
if
err
!=
nil
{
if
err
!=
nil
{
panic
(
err
.
Error
())
panic
(
err
.
Error
())
}
}
consumer
.
MsgNum
=
make
(
chan
int
,
ms
.
rmqBufSize
)
consumer
.
MsgMutex
=
make
(
chan
struct
{},
ms
.
rmqBufSize
)
//consumer.MsgMutex <- struct{}{}
ms
.
consumers
=
append
(
ms
.
consumers
,
*
consumer
)
ms
.
consumers
=
append
(
ms
.
consumers
,
*
consumer
)
ms
.
consumerChannels
=
append
(
ms
.
consumerChannels
,
consumer
.
ChannelName
)
ms
.
consumerChannels
=
append
(
ms
.
consumerChannels
,
consumer
.
ChannelName
)
ms
.
consumerReflects
=
append
(
ms
.
consumerReflects
,
reflect
.
SelectCase
{
ms
.
consumerReflects
=
append
(
ms
.
consumerReflects
,
reflect
.
SelectCase
{
Dir
:
reflect
.
SelectRecv
,
Dir
:
reflect
.
SelectRecv
,
Chan
:
reflect
.
ValueOf
(
consumer
.
Msg
Num
),
Chan
:
reflect
.
ValueOf
(
consumer
.
Msg
Mutex
),
})
})
}
}
}
}
...
@@ -432,25 +439,28 @@ func (ms *RmqTtMsgStream) findTimeTick(consumer rocksmq.Consumer,
...
@@ -432,25 +439,28 @@ func (ms *RmqTtMsgStream) findTimeTick(consumer rocksmq.Consumer,
select
{
select
{
case
<-
ms
.
ctx
.
Done
()
:
case
<-
ms
.
ctx
.
Done
()
:
return
return
case
num
,
ok
:=
<-
consumer
.
MsgNum
:
case
_
,
ok
:=
<-
consumer
.
MsgMutex
:
if
!
ok
{
if
!
ok
{
log
.
Printf
(
"consumer closed!"
)
log
.
Printf
(
"consumer closed!"
)
return
return
}
}
rmqMsg
,
err
:=
rocksmq
.
Rmq
.
Consume
(
consumer
.
GroupName
,
consumer
.
ChannelName
,
num
)
for
{
rmqMsgs
,
err
:=
rocksmq
.
Rmq
.
Consume
(
consumer
.
GroupName
,
consumer
.
ChannelName
,
1
)
if
err
!=
nil
{
if
err
!=
nil
{
log
.
Printf
(
"Failed to consume message in rocksmq, error = %v"
,
err
)
log
.
Printf
(
"Failed to consume message in rocksmq, error = %v"
,
err
)
continue
continue
}
}
if
len
(
rmqMsgs
)
==
0
{
for
j
:=
0
;
j
<
len
(
rmqMsg
);
j
++
{
return
}
rmqMsg
:=
rmqMsgs
[
0
]
headerMsg
:=
commonpb
.
MsgHeader
{}
headerMsg
:=
commonpb
.
MsgHeader
{}
err
:=
proto
.
Unmarshal
(
rmqMsg
[
j
]
.
Payload
,
&
headerMsg
)
err
=
proto
.
Unmarshal
(
rmqMsg
.
Payload
,
&
headerMsg
)
if
err
!=
nil
{
if
err
!=
nil
{
log
.
Printf
(
"Failed to unmarshal message header, error = %v"
,
err
)
log
.
Printf
(
"Failed to unmarshal message header, error = %v"
,
err
)
continue
continue
}
}
tsMsg
,
err
:=
ms
.
unmarshal
.
Unmarshal
(
rmqMsg
[
j
]
.
Payload
,
headerMsg
.
Base
.
MsgType
)
tsMsg
,
err
:=
ms
.
unmarshal
.
Unmarshal
(
rmqMsg
.
Payload
,
headerMsg
.
Base
.
MsgType
)
if
err
!=
nil
{
if
err
!=
nil
{
log
.
Printf
(
"Failed to unmarshal tsMsg, error = %v"
,
err
)
log
.
Printf
(
"Failed to unmarshal tsMsg, error = %v"
,
err
)
continue
continue
...
@@ -458,7 +468,7 @@ func (ms *RmqTtMsgStream) findTimeTick(consumer rocksmq.Consumer,
...
@@ -458,7 +468,7 @@ func (ms *RmqTtMsgStream) findTimeTick(consumer rocksmq.Consumer,
tsMsg
.
SetPosition
(
&
msgstream
.
MsgPosition
{
tsMsg
.
SetPosition
(
&
msgstream
.
MsgPosition
{
ChannelName
:
filepath
.
Base
(
consumer
.
ChannelName
),
ChannelName
:
filepath
.
Base
(
consumer
.
ChannelName
),
MsgID
:
strconv
.
Itoa
(
int
(
rmqMsg
[
j
]
.
MsgID
)),
MsgID
:
strconv
.
Itoa
(
int
(
rmqMsg
.
MsgID
)),
})
})
ms
.
unsolvedMutex
.
Lock
()
ms
.
unsolvedMutex
.
Lock
()
...
@@ -469,7 +479,8 @@ func (ms *RmqTtMsgStream) findTimeTick(consumer rocksmq.Consumer,
...
@@ -469,7 +479,8 @@ func (ms *RmqTtMsgStream) findTimeTick(consumer rocksmq.Consumer,
findMapMutex
.
Lock
()
findMapMutex
.
Lock
()
eofMsgMap
[
consumer
]
=
tsMsg
.
(
*
TimeTickMsg
)
.
Base
.
Timestamp
eofMsgMap
[
consumer
]
=
tsMsg
.
(
*
TimeTickMsg
)
.
Base
.
Timestamp
findMapMutex
.
Unlock
()
findMapMutex
.
Unlock
()
return
//consumer.MsgMutex <- struct{}{}
//return
}
}
}
}
}
}
...
@@ -504,8 +515,8 @@ func (ms *RmqTtMsgStream) Seek(mp *msgstream.MsgPosition) error {
...
@@ -504,8 +515,8 @@ func (ms *RmqTtMsgStream) Seek(mp *msgstream.MsgPosition) error {
ms
.
unsolvedMutex
.
Lock
()
ms
.
unsolvedMutex
.
Lock
()
ms
.
unsolvedBuf
[
consumer
]
=
make
([]
TsMsg
,
0
)
ms
.
unsolvedBuf
[
consumer
]
=
make
([]
TsMsg
,
0
)
// When rmq seek is called, msg
Num
can't be used before current msgs all consumed, because
// When rmq seek is called, msg
Mutex
can't be used before current msgs all consumed, because
// new msg
Num
is not generated. So just try to consume msgs
// new msg
Mutex
is not generated. So just try to consume msgs
for
{
for
{
rmqMsg
,
err
:=
rocksmq
.
Rmq
.
Consume
(
consumer
.
GroupName
,
consumer
.
ChannelName
,
1
)
rmqMsg
,
err
:=
rocksmq
.
Rmq
.
Consume
(
consumer
.
GroupName
,
consumer
.
ChannelName
,
1
)
if
err
!=
nil
{
if
err
!=
nil
{
...
...
internal/util/rocksmq/global_rmq.go
浏览文件 @
0d499a9f
...
@@ -15,7 +15,7 @@ var once sync.Once
...
@@ -15,7 +15,7 @@ var once sync.Once
type
Consumer
struct
{
type
Consumer
struct
{
GroupName
string
GroupName
string
ChannelName
string
ChannelName
string
Msg
Num
chan
int
Msg
Mutex
chan
struct
{}
}
}
func
InitRmq
(
rocksdbName
string
,
idAllocator
allocator
.
GIDAllocator
)
error
{
func
InitRmq
(
rocksdbName
string
,
idAllocator
allocator
.
GIDAllocator
)
error
{
...
...
internal/util/rocksmq/rocksmq.go
浏览文件 @
0d499a9f
...
@@ -247,8 +247,8 @@ func (rmq *RocksMQ) Produce(channelName string, messages []ProducerMessage) erro
...
@@ -247,8 +247,8 @@ func (rmq *RocksMQ) Produce(channelName string, messages []ProducerMessage) erro
}
}
for
_
,
consumer
:=
range
rmq
.
notify
[
channelName
]
{
for
_
,
consumer
:=
range
rmq
.
notify
[
channelName
]
{
if
consumer
.
Msg
Num
!=
nil
{
if
consumer
.
Msg
Mutex
!=
nil
{
consumer
.
Msg
Num
<-
msgLen
consumer
.
Msg
Mutex
<-
struct
{}{}
}
}
}
}
return
nil
return
nil
...
@@ -308,6 +308,7 @@ func (rmq *RocksMQ) Consume(groupName string, channelName string, n int) ([]Cons
...
@@ -308,6 +308,7 @@ func (rmq *RocksMQ) Consume(groupName string, channelName string, n int) ([]Cons
return
nil
,
err
return
nil
,
err
}
}
// When already consume to last mes, an empty slice will be returned
if
len
(
consumerMessage
)
==
0
{
if
len
(
consumerMessage
)
==
0
{
return
consumerMessage
,
nil
return
consumerMessage
,
nil
}
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录