Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
85bc2a54
M
milvus
项目概览
milvus
/
milvus
大约 1 年 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
85bc2a54
编写于
3月 25, 2021
作者:
Y
yukun
提交者:
yefu.chen
3月 25, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Fix load twice hang bug
Signed-off-by:
N
yukun
<
kun.yu@zilliz.com
>
上级
1e4afda8
变更
5
隐藏空白更改
内联
并排
Showing
5 changed file
with
112 addition
and
31 deletion
+112
-31
internal/msgstream/rmqms/rmq_msgstream.go
internal/msgstream/rmqms/rmq_msgstream.go
+1
-0
internal/util/rocksmq/client/rocksmq/client_impl.go
internal/util/rocksmq/client/rocksmq/client_impl.go
+61
-31
internal/util/rocksmq/client/rocksmq/consumer_impl.go
internal/util/rocksmq/client/rocksmq/consumer_impl.go
+28
-0
internal/util/rocksmq/server/rocksmq/rocksmq.go
internal/util/rocksmq/server/rocksmq/rocksmq.go
+1
-0
internal/util/rocksmq/server/rocksmq/rocksmq_impl.go
internal/util/rocksmq/server/rocksmq/rocksmq_impl.go
+21
-0
未找到文件。
internal/msgstream/rmqms/rmq_msgstream.go
浏览文件 @
85bc2a54
...
...
@@ -597,6 +597,7 @@ func (rtms *RmqTtMsgStream) Seek(mp *msgstream.MsgPosition) error {
rtms
.
addConsumer
(
consumer
,
seekChannel
)
if
len
(
consumer
.
Chan
())
==
0
{
log
.
Debug
(
"len(consumer.Chan()) = 0"
)
return
nil
}
...
...
internal/util/rocksmq/client/rocksmq/client_impl.go
浏览文件 @
85bc2a54
package
rocksmq
import
(
"strconv"
"github.com/zilliztech/milvus-distributed/internal/log"
server
"github.com/zilliztech/milvus-distributed/internal/util/rocksmq/server/rocksmq"
)
...
...
@@ -44,6 +42,22 @@ func (c *client) CreateProducer(options ProducerOptions) (Producer, error) {
func
(
c
*
client
)
Subscribe
(
options
ConsumerOptions
)
(
Consumer
,
error
)
{
// Create a consumer
//for _, con := range c.consumers {
// log.Debug(con.Topic() + "---------------" + con.Subscription())
// if con.Topic() == options.Topic && con.Subscription() == options.SubscriptionName {
// log.Debug("consumer existed")
// return con, nil
// }
//}
if
exist
,
con
:=
c
.
server
.
ExistConsumerGroup
(
options
.
Topic
,
options
.
SubscriptionName
);
exist
{
log
.
Debug
(
"EXISTED"
)
consumer
,
err
:=
newConsumer1
(
c
,
options
,
con
.
MsgMutex
)
if
err
!=
nil
{
return
nil
,
err
}
go
consume
(
consumer
)
return
consumer
,
nil
}
consumer
,
err
:=
newConsumer
(
c
,
options
)
if
err
!=
nil
{
return
nil
,
err
...
...
@@ -65,42 +79,58 @@ func (c *client) Subscribe(options ConsumerOptions) (Consumer, error) {
// Take messages from RocksDB and put it into consumer.Chan(),
// trigger by consumer.MsgMutex which trigger by producer
go
func
()
{
for
{
//nolint:gosimple
select
{
case
_
,
ok
:=
<-
consumer
.
MsgMutex
()
:
if
!
ok
{
// consumer MsgMutex closed, goroutine exit
return
go
consume
(
consumer
)
c
.
consumerOptions
=
append
(
c
.
consumerOptions
,
options
)
return
consumer
,
nil
}
func
consume
(
consumer
*
consumer
)
{
for
{
//nolint:gosimple
log
.
Debug
(
consumer
.
topic
+
"+"
+
consumer
.
consumerName
)
//if consumer.msgMutex == nil {
// break
//}
select
{
//nolint:gosimple
case
_
,
ok
:=
<-
consumer
.
MsgMutex
()
:
if
!
ok
{
// consumer MsgMutex closed, goroutine exit
log
.
Debug
(
"consumer MsgMutex closed"
)
return
}
log
.
Debug
(
"Before consume"
)
for
{
msg
,
err
:=
consumer
.
client
.
server
.
Consume
(
consumer
.
topic
,
consumer
.
consumerName
,
1
)
if
err
!=
nil
{
log
.
Debug
(
"Consumer's goroutine cannot consume from ("
+
consumer
.
topic
+
","
+
consumer
.
consumerName
+
"): "
+
err
.
Error
())
break
}
if
len
(
msg
)
!=
1
{
//log.Debug("Consumer's goroutine cannot consume from (" + consumer.topic +
// "," + consumer.consumerName + "): message len(" + strconv.Itoa(len(msg)) +
// ") is not 1")
break
}
for
{
msg
,
err
:=
consumer
.
client
.
server
.
Consume
(
consumer
.
topic
,
consumer
.
consumerName
,
1
)
if
err
!=
nil
{
log
.
Debug
(
"Consumer's goroutine cannot consume from ("
+
consumer
.
topic
+
","
+
consumer
.
consumerName
+
"): "
+
err
.
Error
())
break
}
if
len
(
msg
)
!=
1
{
log
.
Debug
(
"Consumer's goroutine cannot consume from ("
+
consumer
.
topic
+
","
+
consumer
.
consumerName
+
"): message len("
+
strconv
.
Itoa
(
len
(
msg
))
+
") is not 1"
)
break
}
consumer
.
messageCh
<-
ConsumerMessage
{
MsgID
:
msg
[
0
]
.
MsgID
,
Payload
:
msg
[
0
]
.
Payload
,
}
consumer
.
messageCh
<-
ConsumerMessage
{
MsgID
:
msg
[
0
]
.
MsgID
,
Payload
:
msg
[
0
]
.
Payload
,
}
}
//default:
// log.Debug("In default")
}
}()
return
consumer
,
nil
}
}
func
(
c
*
client
)
Close
()
{
// TODO: free resources
for
_
,
opt
:=
range
c
.
consumerOptions
{
log
.
Debug
(
"Close"
+
opt
.
Topic
+
"+"
+
opt
.
SubscriptionName
)
_
=
c
.
server
.
DestroyConsumerGroup
(
opt
.
Topic
,
opt
.
SubscriptionName
)
//_ = c.server.DestroyTopic(opt.Topic)
}
}
internal/util/rocksmq/client/rocksmq/consumer_impl.go
浏览文件 @
85bc2a54
...
...
@@ -38,6 +38,34 @@ func newConsumer(c *client, options ConsumerOptions) (*consumer, error) {
},
nil
}
func
newConsumer1
(
c
*
client
,
options
ConsumerOptions
,
msgMutex
chan
struct
{})
(
*
consumer
,
error
)
{
if
c
==
nil
{
return
nil
,
newError
(
InvalidConfiguration
,
"client is nil"
)
}
if
options
.
Topic
==
""
{
return
nil
,
newError
(
InvalidConfiguration
,
"Topic is empty"
)
}
if
options
.
SubscriptionName
==
""
{
return
nil
,
newError
(
InvalidConfiguration
,
"SubscriptionName is empty"
)
}
messageCh
:=
options
.
MessageChannel
if
options
.
MessageChannel
==
nil
{
messageCh
=
make
(
chan
ConsumerMessage
,
1
)
}
return
&
consumer
{
topic
:
options
.
Topic
,
client
:
c
,
consumerName
:
options
.
SubscriptionName
,
options
:
options
,
msgMutex
:
msgMutex
,
messageCh
:
messageCh
,
},
nil
}
func
(
c
*
consumer
)
Subscription
()
string
{
return
c
.
consumerName
}
...
...
internal/util/rocksmq/server/rocksmq/rocksmq.go
浏览文件 @
85bc2a54
...
...
@@ -26,4 +26,5 @@ type RocksMQ interface {
Produce
(
topicName
string
,
messages
[]
ProducerMessage
)
error
Consume
(
topicName
string
,
groupName
string
,
n
int
)
([]
ConsumerMessage
,
error
)
Seek
(
topicName
string
,
groupName
string
,
msgID
UniqueID
)
error
ExistConsumerGroup
(
topicName
string
,
groupName
string
)
(
bool
,
*
Consumer
)
}
internal/util/rocksmq/server/rocksmq/rocksmq_impl.go
浏览文件 @
85bc2a54
...
...
@@ -134,10 +134,23 @@ func (rmq *rocksmq) DestroyTopic(topicName string) error {
log
.
Debug
(
"RocksMQ: remove "
+
endKey
+
" failed."
)
return
err
}
log
.
Debug
(
"DestroyTopic: "
+
topicName
)
return
nil
}
func
(
rmq
*
rocksmq
)
ExistConsumerGroup
(
topicName
,
groupName
string
)
(
bool
,
*
Consumer
)
{
key
:=
groupName
+
"/"
+
topicName
+
"/current_id"
if
rmq
.
checkKeyExist
(
key
)
{
for
_
,
con
:=
range
rmq
.
consumers
[
topicName
]
{
if
con
.
GroupName
==
groupName
{
return
true
,
con
}
}
}
return
false
,
nil
}
func
(
rmq
*
rocksmq
)
CreateConsumerGroup
(
topicName
,
groupName
string
)
error
{
key
:=
groupName
+
"/"
+
topicName
+
"/current_id"
if
rmq
.
checkKeyExist
(
key
)
{
...
...
@@ -170,6 +183,14 @@ func (rmq *rocksmq) DestroyConsumerGroup(topicName, groupName string) error {
log
.
Debug
(
"RocksMQ: remove "
+
key
+
" failed."
)
return
err
}
for
index
,
con
:=
range
rmq
.
consumers
[
topicName
]
{
if
con
.
GroupName
==
groupName
{
rmq
.
consumers
[
topicName
]
=
append
(
rmq
.
consumers
[
topicName
][
:
index
],
rmq
.
consumers
[
topicName
][
index
+
1
:
]
...
)
}
}
log
.
Debug
(
"DestroyConsumerGroup: "
+
topicName
+
"+"
+
groupName
)
return
nil
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录