Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
e3fadc45
M
milvus
项目概览
milvus
/
milvus
12 个月 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
e3fadc45
编写于
2月 04, 2021
作者:
Y
yukun
提交者:
yefu.chen
2月 04, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Fix for new msgstream interface
Signed-off-by:
N
yukun
<
kun.yu@zilliz.com
>
上级
b89e5a32
变更
8
隐藏空白更改
内联
并排
Showing
8 changed file
with
111 addition
and
30 deletion
+111
-30
internal/dataservice/meta.go
internal/dataservice/meta.go
+1
-1
internal/dataservice/segment_allocator.go
internal/dataservice/segment_allocator.go
+2
-2
internal/dataservice/segment_allocator_test.go
internal/dataservice/segment_allocator_test.go
+3
-6
internal/dataservice/server.go
internal/dataservice/server.go
+1
-4
internal/dataservice/watcher_test.go
internal/dataservice/watcher_test.go
+1
-1
internal/msgstream/pulsarms/pulsar_msgstream.go
internal/msgstream/pulsarms/pulsar_msgstream.go
+1
-1
internal/msgstream/rmqms/factory.go
internal/msgstream/rmqms/factory.go
+28
-0
internal/msgstream/rmqms/rmq_msgstream.go
internal/msgstream/rmqms/rmq_msgstream.go
+74
-15
未找到文件。
internal/dataservice/meta.go
浏览文件 @
e3fadc45
...
...
@@ -183,7 +183,7 @@ func (meta *meta) UpdateSegment(segmentInfo *datapb.SegmentInfo) error {
func
(
meta
*
meta
)
DropSegment
(
segmentID
UniqueID
)
error
{
meta
.
ddLock
.
Lock
()
meta
.
ddLock
.
Unlock
()
defer
meta
.
ddLock
.
Unlock
()
if
_
,
ok
:=
meta
.
segID2Info
[
segmentID
];
!
ok
{
return
newErrSegmentNotFound
(
segmentID
)
...
...
internal/dataservice/segment_allocator.go
浏览文件 @
e3fadc45
...
...
@@ -71,7 +71,7 @@ type (
}
)
func
newSegmentAllocator
(
meta
*
meta
,
allocator
allocator
)
(
*
segmentAllocatorImpl
,
error
)
{
func
newSegmentAllocator
(
meta
*
meta
,
allocator
allocator
)
*
segmentAllocatorImpl
{
segmentAllocator
:=
&
segmentAllocatorImpl
{
mt
:
meta
,
segments
:
make
(
map
[
UniqueID
]
*
segmentStatus
),
...
...
@@ -80,7 +80,7 @@ func newSegmentAllocator(meta *meta, allocator allocator) (*segmentAllocatorImpl
segmentThresholdFactor
:
Params
.
SegmentSizeFactor
,
allocator
:
allocator
,
}
return
segmentAllocator
,
nil
return
segmentAllocator
}
func
(
allocator
*
segmentAllocatorImpl
)
OpenSegment
(
segmentInfo
*
datapb
.
SegmentInfo
)
error
{
...
...
internal/dataservice/segment_allocator_test.go
浏览文件 @
e3fadc45
...
...
@@ -17,8 +17,7 @@ func TestAllocSegment(t *testing.T) {
mockAllocator
:=
newMockAllocator
()
meta
,
err
:=
newMemoryMeta
(
mockAllocator
)
assert
.
Nil
(
t
,
err
)
segAllocator
,
err
:=
newSegmentAllocator
(
meta
,
mockAllocator
)
assert
.
Nil
(
t
,
err
)
segAllocator
:=
newSegmentAllocator
(
meta
,
mockAllocator
)
schema
:=
newTestSchema
()
collID
,
err
:=
mockAllocator
.
allocID
()
...
...
@@ -68,8 +67,7 @@ func TestSealSegment(t *testing.T) {
mockAllocator
:=
newMockAllocator
()
meta
,
err
:=
newMemoryMeta
(
mockAllocator
)
assert
.
Nil
(
t
,
err
)
segAllocator
,
err
:=
newSegmentAllocator
(
meta
,
mockAllocator
)
assert
.
Nil
(
t
,
err
)
segAllocator
:=
newSegmentAllocator
(
meta
,
mockAllocator
)
schema
:=
newTestSchema
()
collID
,
err
:=
mockAllocator
.
allocID
()
...
...
@@ -105,8 +103,7 @@ func TestExpireSegment(t *testing.T) {
mockAllocator
:=
newMockAllocator
()
meta
,
err
:=
newMemoryMeta
(
mockAllocator
)
assert
.
Nil
(
t
,
err
)
segAllocator
,
err
:=
newSegmentAllocator
(
meta
,
mockAllocator
)
assert
.
Nil
(
t
,
err
)
segAllocator
:=
newSegmentAllocator
(
meta
,
mockAllocator
)
schema
:=
newTestSchema
()
collID
,
err
:=
mockAllocator
.
allocID
()
...
...
internal/dataservice/server.go
浏览文件 @
e3fadc45
...
...
@@ -134,10 +134,7 @@ func (s *Server) Start() error {
return
err
}
s
.
statsHandler
=
newStatsHandler
(
s
.
meta
)
s
.
segAllocator
,
err
=
newSegmentAllocator
(
s
.
meta
,
s
.
allocator
)
if
err
!=
nil
{
return
err
}
s
.
segAllocator
=
newSegmentAllocator
(
s
.
meta
,
s
.
allocator
)
s
.
ddHandler
=
newDDHandler
(
s
.
meta
,
s
.
segAllocator
)
s
.
initSegmentInfoChannel
()
if
err
=
s
.
loadMetaFromMaster
();
err
!=
nil
{
...
...
internal/dataservice/watcher_test.go
浏览文件 @
e3fadc45
...
...
@@ -21,7 +21,7 @@ func TestDataNodeTTWatcher(t *testing.T) {
allocator
:=
newMockAllocator
()
meta
,
err
:=
newMemoryMeta
(
allocator
)
assert
.
Nil
(
t
,
err
)
segAllocator
,
err
:=
newSegmentAllocator
(
meta
,
allocator
)
segAllocator
:=
newSegmentAllocator
(
meta
,
allocator
)
assert
.
Nil
(
t
,
err
)
watcher
:=
newDataNodeTimeTickWatcher
(
meta
,
segAllocator
,
cluster
)
...
...
internal/msgstream/pulsarms/pulsar_msgstream.go
浏览文件 @
e3fadc45
...
...
@@ -747,7 +747,7 @@ func checkTimeTickMsg(msg map[Consumer]Timestamp,
for
consumer
:=
range
msg
{
mu
.
RLock
()
v
:=
msg
[
consumer
]
mu
.
Unlock
()
mu
.
R
Unlock
()
if
v
!=
maxTime
{
isChannelReady
[
consumer
]
=
false
}
else
{
...
...
internal/msgstream/rmqms/factory.go
0 → 100644
浏览文件 @
e3fadc45
package
rmqms
import
(
"context"
"github.com/zilliztech/milvus-distributed/internal/msgstream"
)
type
Factory
struct
{
dispatcherFactory
msgstream
.
ProtoUDFactory
address
string
receiveBufSize
int64
pulsarBufSize
int64
}
func
(
f
*
Factory
)
NewMsgStream
(
ctx
context
.
Context
)
(
msgstream
.
MsgStream
,
error
)
{
return
newRmqMsgStream
(
ctx
,
f
.
receiveBufSize
,
f
.
dispatcherFactory
.
NewUnmarshalDispatcher
())
}
func
NewFactory
(
address
string
,
receiveBufSize
int64
,
pulsarBufSize
int64
)
*
Factory
{
f
:=
&
Factory
{
dispatcherFactory
:
msgstream
.
ProtoUDFactory
{},
address
:
address
,
receiveBufSize
:
receiveBufSize
,
pulsarBufSize
:
pulsarBufSize
,
}
return
f
}
internal/msgstream/rmqms/rmq_msgstream.go
浏览文件 @
e3fadc45
...
...
@@ -16,6 +16,17 @@ import (
"github.com/zilliztech/milvus-distributed/internal/msgstream"
)
type
TsMsg
=
msgstream
.
TsMsg
type
MsgPack
=
msgstream
.
MsgPack
type
MsgType
=
msgstream
.
MsgType
type
UniqueID
=
msgstream
.
UniqueID
type
BaseMsg
=
msgstream
.
BaseMsg
type
Timestamp
=
msgstream
.
Timestamp
type
IntPrimaryKey
=
msgstream
.
IntPrimaryKey
type
TimeTickMsg
=
msgstream
.
TimeTickMsg
type
QueryNodeStatsMsg
=
msgstream
.
QueryNodeStatsMsg
type
RepackFunc
=
msgstream
.
RepackFunc
type
RmqMsgStream
struct
{
isServing
int64
ctx
context
.
Context
...
...
@@ -23,7 +34,6 @@ type RmqMsgStream struct {
serverLoopCtx
context
.
Context
serverLoopCancel
func
()
rmq
*
rocksmq
.
RocksMQ
repackFunc
msgstream
.
RepackFunc
consumers
[]
rocksmq
.
Consumer
producers
[]
string
...
...
@@ -35,17 +45,18 @@ type RmqMsgStream struct {
streamCancel
func
()
}
func
NewRmqMsgStream
(
ctx
context
.
Context
,
rmq
*
rocksmq
.
RocksMQ
,
receiveBufSize
int64
)
*
RmqMsgStream
{
func
newRmqMsgStream
(
ctx
context
.
Context
,
receiveBufSize
int64
,
unmarshal
msgstream
.
UnmarshalDispatcher
)
(
*
RmqMsgStream
,
error
)
{
streamCtx
,
streamCancel
:=
context
.
WithCancel
(
ctx
)
receiveBuf
:=
make
(
chan
*
msgstream
.
MsgPack
,
receiveBufSize
)
stream
:=
&
RmqMsgStream
{
ctx
:
streamCtx
,
rmq
:
nil
,
receiveBuf
:
receiveBuf
,
unmarshal
:
unmarshal
,
streamCancel
:
streamCancel
,
}
return
stream
return
stream
,
nil
}
func
(
ms
*
RmqMsgStream
)
Start
()
{
...
...
@@ -59,25 +70,32 @@ func (ms *RmqMsgStream) Start() {
func
(
ms
*
RmqMsgStream
)
Close
()
{
}
func
(
ms
*
RmqMsgStream
)
CreateProducers
(
channels
[]
string
)
error
{
type
propertiesReaderWriter
struct
{
ppMap
map
[
string
]
string
}
func
(
ms
*
RmqMsgStream
)
SetRepackFunc
(
repackFunc
RepackFunc
)
{
ms
.
repackFunc
=
repackFunc
}
func
(
ms
*
RmqMsgStream
)
AsProducer
(
channels
[]
string
)
{
for
_
,
channel
:=
range
channels
{
// TODO(yhz): Here may allow to create an existing channel
if
err
:=
ms
.
rmq
.
CreateChannel
(
channel
);
err
!=
nil
{
return
err
if
err
:=
rocksmq
.
Rmq
.
CreateChannel
(
channel
);
err
!=
nil
{
errMsg
:=
"Failed to create producer "
+
channel
+
", error = "
+
err
.
Error
()
panic
(
errMsg
)
}
}
return
nil
}
func
(
ms
*
RmqMsgStream
)
CreateConsumers
(
channels
[]
string
,
groupName
string
)
error
{
func
(
ms
*
RmqMsgStream
)
AsConsumer
(
channels
[]
string
,
groupName
string
)
{
for
_
,
channelName
:=
range
channels
{
if
err
:=
ms
.
r
mq
.
CreateConsumerGroup
(
groupName
,
channelName
);
err
!=
nil
{
return
err
if
err
:=
rocksmq
.
R
mq
.
CreateConsumerGroup
(
groupName
,
channelName
);
err
!=
nil
{
panic
(
err
.
Error
())
}
msgNum
:=
make
(
chan
int
)
ms
.
consumers
=
append
(
ms
.
consumers
,
rocksmq
.
Consumer
{
GroupName
:
groupName
,
ChannelName
:
channelName
,
MsgNum
:
msgNum
})
}
return
nil
}
func
(
ms
*
RmqMsgStream
)
Produce
(
pack
*
msgstream
.
MsgPack
)
error
{
...
...
@@ -172,7 +190,30 @@ func (ms *RmqMsgStream) Produce(pack *msgstream.MsgPack) error {
}
msg
:=
make
([]
rocksmq
.
ProducerMessage
,
0
)
msg
=
append
(
msg
,
*
rocksmq
.
NewProducerMessage
(
m
))
if
err
:=
ms
.
rmq
.
Produce
(
ms
.
producers
[
k
],
msg
);
err
!=
nil
{
if
err
:=
rocksmq
.
Rmq
.
Produce
(
ms
.
producers
[
k
],
msg
);
err
!=
nil
{
return
err
}
}
}
return
nil
}
func
(
ms
*
RmqMsgStream
)
Broadcast
(
msgPack
*
MsgPack
)
error
{
producerLen
:=
len
(
ms
.
producers
)
for
_
,
v
:=
range
msgPack
.
Msgs
{
mb
,
err
:=
v
.
Marshal
(
v
)
if
err
!=
nil
{
return
err
}
m
,
err
:=
msgstream
.
ConvertToByteArray
(
mb
)
if
err
!=
nil
{
return
err
}
msg
:=
make
([]
rocksmq
.
ProducerMessage
,
0
)
msg
=
append
(
msg
,
*
rocksmq
.
NewProducerMessage
(
m
))
for
i
:=
0
;
i
<
producerLen
;
i
++
{
if
err
:=
rocksmq
.
Rmq
.
Produce
(
ms
.
producers
[
i
],
msg
);
err
!=
nil
{
return
err
}
}
...
...
@@ -221,7 +262,7 @@ func (ms *RmqMsgStream) bufMsgPackToChannel() {
}
msgNum
:=
value
.
Interface
()
.
(
int
)
rmqMsg
,
err
:=
ms
.
r
mq
.
Consume
(
ms
.
consumers
[
chosen
]
.
GroupName
,
ms
.
consumers
[
chosen
]
.
ChannelName
,
msgNum
)
rmqMsg
,
err
:=
rocksmq
.
R
mq
.
Consume
(
ms
.
consumers
[
chosen
]
.
GroupName
,
ms
.
consumers
[
chosen
]
.
ChannelName
,
msgNum
)
if
err
!=
nil
{
log
.
Printf
(
"Failed to consume message in rocksmq, error = %v"
,
err
)
continue
...
...
@@ -261,5 +302,23 @@ func (ms *RmqMsgStream) bufMsgPackToChannel() {
}
func
(
ms
*
RmqMsgStream
)
Chan
()
<-
chan
*
msgstream
.
MsgPack
{
return
nil
return
ms
.
receiveBuf
}
func
(
ms
*
RmqMsgStream
)
Seek
(
offset
*
msgstream
.
MsgPosition
)
error
{
for
i
:=
0
;
i
<
len
(
ms
.
consumers
);
i
++
{
if
ms
.
consumers
[
i
]
.
ChannelName
==
offset
.
ChannelName
{
messageID
,
err
:=
strconv
.
ParseInt
(
offset
.
MsgID
,
10
,
64
)
if
err
!=
nil
{
return
err
}
err
=
rocksmq
.
Rmq
.
Seek
(
ms
.
consumers
[
i
]
.
GroupName
,
ms
.
consumers
[
i
]
.
ChannelName
,
messageID
)
if
err
!=
nil
{
return
err
}
return
nil
}
}
return
errors
.
New
(
"msgStream seek fail"
)
}
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录