Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
78bc688d
M
milvus
项目概览
milvus
/
milvus
10 个月 前同步成功
通知
260
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
前往新版Gitcode,体验更适合开发者的 AI 搜索 >>
未验证
提交
78bc688d
编写于
8月 16, 2023
作者:
E
Enwei Jiao
提交者:
GitHub
8月 16, 2023
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Remove QueryMsgStream in MqFactory interface (#26374)
Signed-off-by:
N
Enwei Jiao
<
enwei.jiao@zilliz.com
>
上级
4742049e
变更
14
隐藏空白更改
内联
并排
Showing
14 changed file
with
7 addition
and
90 deletion
+7
-90
docs/developer_guides/chap04_message_stream.md
docs/developer_guides/chap04_message_stream.md
+0
-1
internal/datanode/flow_graph_dmstream_input_node_test.go
internal/datanode/flow_graph_dmstream_input_node_test.go
+0
-4
internal/indexnode/chunkmgr_mock.go
internal/indexnode/chunkmgr_mock.go
+0
-5
internal/mq/msgstream/mq_factory_test.go
internal/mq/msgstream/mq_factory_test.go
+0
-5
internal/proxy/channels_mgr.go
internal/proxy/channels_mgr.go
+4
-19
internal/proxy/channels_mgr_test.go
internal/proxy/channels_mgr_test.go
+3
-6
internal/proxy/mock_msgstream_test.go
internal/proxy/mock_msgstream_test.go
+0
-7
internal/proxy/mock_test.go
internal/proxy/mock_test.go
+0
-4
internal/util/dependency/factory.go
internal/util/dependency/factory.go
+0
-4
pkg/mq/msgstream/common_mq_factory.go
pkg/mq/msgstream/common_mq_factory.go
+0
-10
pkg/mq/msgstream/factory_test.go
pkg/mq/msgstream/factory_test.go
+0
-3
pkg/mq/msgstream/mq_factory.go
pkg/mq/msgstream/mq_factory.go
+0
-9
pkg/mq/msgstream/mq_factory_test.go
pkg/mq/msgstream/mq_factory_test.go
+0
-12
pkg/mq/msgstream/msgstream.go
pkg/mq/msgstream/msgstream.go
+0
-1
未找到文件。
docs/developer_guides/chap04_message_stream.md
浏览文件 @
78bc688d
...
...
@@ -216,7 +216,6 @@ type Factory interface {
Init
(
params
*
paramtable
.
ComponentParam
)
error
NewMsgStream
(
ctx
context
.
Context
)
(
MsgStream
,
error
)
NewTtMsgStream
(
ctx
context
.
Context
)
(
MsgStream
,
error
)
NewQueryMsgStream
(
ctx
context
.
Context
)
(
MsgStream
,
error
)
}
// Pulsar
...
...
internal/datanode/flow_graph_dmstream_input_node_test.go
浏览文件 @
78bc688d
...
...
@@ -56,10 +56,6 @@ func (mm *mockMsgStreamFactory) NewTtMsgStream(ctx context.Context) (msgstream.M
return
&
mockTtMsgStream
{},
nil
}
func
(
mm
*
mockMsgStreamFactory
)
NewQueryMsgStream
(
ctx
context
.
Context
)
(
msgstream
.
MsgStream
,
error
)
{
return
nil
,
nil
}
func
(
mm
*
mockMsgStreamFactory
)
NewMsgStreamDisposer
(
ctx
context
.
Context
)
func
([]
string
,
string
)
error
{
return
nil
}
...
...
internal/indexnode/chunkmgr_mock.go
浏览文件 @
78bc688d
...
...
@@ -245,11 +245,6 @@ func (f *mockFactory) NewTtMsgStream(context.Context) (msgstream.MsgStream, erro
return
nil
,
errNotImplErr
}
func
(
f
*
mockFactory
)
NewQueryMsgStream
(
context
.
Context
)
(
msgstream
.
MsgStream
,
error
)
{
// TODO
return
nil
,
errNotImplErr
}
func
(
f
*
mockFactory
)
NewMsgStreamDisposer
(
ctx
context
.
Context
)
func
([]
string
,
string
)
error
{
// TODO
return
nil
...
...
internal/mq/msgstream/mq_factory_test.go
浏览文件 @
78bc688d
...
...
@@ -41,9 +41,4 @@ func TestRmsFactory(t *testing.T) {
_
,
err
=
rmsFactory
.
NewTtMsgStream
(
ctx
)
assert
.
NoError
(
t
,
err
)
_
,
err
=
rmsFactory
.
NewQueryMsgStream
(
ctx
)
assert
.
NoError
(
t
,
err
)
err
=
rmsFactory
.
NewMsgStreamDisposer
(
ctx
)([]
string
{
"hello"
},
"xx"
)
assert
.
NoError
(
t
,
err
)
}
internal/proxy/channels_mgr.go
浏览文件 @
78bc688d
...
...
@@ -113,21 +113,12 @@ func getDmlChannelsFunc(ctx context.Context, rc types.RootCoord) getChannelsFunc
}
}
// streamType indicates which type of message stream should be created.
type
streamType
int
const
(
dmlStreamType
streamType
=
iota
dqlStreamType
)
type
singleTypeChannelsMgr
struct
{
infos
map
[
UniqueID
]
streamInfos
// collection id -> stream infos
mu
sync
.
RWMutex
getChannelsFunc
getChannelsFuncType
repackFunc
repackFuncType
singleStreamType
streamType
msgStreamFactory
msgstream
.
Factory
}
...
...
@@ -184,15 +175,11 @@ func (mgr *singleTypeChannelsMgr) streamExistPrivate(collectionID UniqueID) bool
return
ok
&&
streamInfos
.
stream
!=
nil
}
func
createStream
(
factory
msgstream
.
Factory
,
streamType
streamType
,
pchans
[]
pChan
,
repack
repackFuncType
)
(
msgstream
.
MsgStream
,
error
)
{
func
createStream
(
factory
msgstream
.
Factory
,
pchans
[]
pChan
,
repack
repackFuncType
)
(
msgstream
.
MsgStream
,
error
)
{
var
stream
msgstream
.
MsgStream
var
err
error
if
streamType
==
dqlStreamType
{
stream
,
err
=
factory
.
NewQueryMsgStream
(
context
.
Background
())
}
else
{
stream
,
err
=
factory
.
NewMsgStream
(
context
.
Background
())
}
stream
,
err
=
factory
.
NewMsgStream
(
context
.
Background
())
if
err
!=
nil
{
return
nil
,
err
...
...
@@ -240,7 +227,7 @@ func (mgr *singleTypeChannelsMgr) createMsgStream(collectionID UniqueID) (msgstr
return
nil
,
err
}
stream
,
err
:=
createStream
(
mgr
.
msgStreamFactory
,
mgr
.
singleStreamType
,
channelInfos
.
pchans
,
mgr
.
repackFunc
)
stream
,
err
:=
createStream
(
mgr
.
msgStreamFactory
,
channelInfos
.
pchans
,
mgr
.
repackFunc
)
if
err
!=
nil
{
// What if stream created by other goroutines?
log
.
Error
(
"failed to create message stream"
,
zap
.
Error
(
err
),
zap
.
Int64
(
"collection"
,
collectionID
))
...
...
@@ -309,13 +296,11 @@ func newSingleTypeChannelsMgr(
getChannelsFunc
getChannelsFuncType
,
msgStreamFactory
msgstream
.
Factory
,
repackFunc
repackFuncType
,
singleStreamType
streamType
,
)
*
singleTypeChannelsMgr
{
return
&
singleTypeChannelsMgr
{
infos
:
make
(
map
[
UniqueID
]
streamInfos
),
getChannelsFunc
:
getChannelsFunc
,
repackFunc
:
repackFunc
,
singleStreamType
:
singleStreamType
,
msgStreamFactory
:
msgStreamFactory
,
}
}
...
...
@@ -355,6 +340,6 @@ func newChannelsMgrImpl(
msgStreamFactory
msgstream
.
Factory
,
)
*
channelsMgrImpl
{
return
&
channelsMgrImpl
{
dmlChannelsMgr
:
newSingleTypeChannelsMgr
(
getDmlChannelsFunc
,
msgStreamFactory
,
dmlRepackFunc
,
dmlStreamType
),
dmlChannelsMgr
:
newSingleTypeChannelsMgr
(
getDmlChannelsFunc
,
msgStreamFactory
,
dmlRepackFunc
),
}
}
internal/proxy/channels_mgr_test.go
浏览文件 @
78bc688d
...
...
@@ -213,7 +213,7 @@ func Test_createStream(t *testing.T) {
factory
.
fQStream
=
func
(
ctx
context
.
Context
)
(
msgstream
.
MsgStream
,
error
)
{
return
nil
,
errors
.
New
(
"mock"
)
}
_
,
err
:=
createStream
(
factory
,
dmlStreamType
,
nil
,
nil
)
_
,
err
:=
createStream
(
factory
,
nil
,
nil
)
assert
.
Error
(
t
,
err
)
})
...
...
@@ -222,7 +222,7 @@ func Test_createStream(t *testing.T) {
factory
.
f
=
func
(
ctx
context
.
Context
)
(
msgstream
.
MsgStream
,
error
)
{
return
nil
,
errors
.
New
(
"mock"
)
}
_
,
err
:=
createStream
(
factory
,
dqlStreamType
,
nil
,
nil
)
_
,
err
:=
createStream
(
factory
,
nil
,
nil
)
assert
.
Error
(
t
,
err
)
})
...
...
@@ -231,7 +231,7 @@ func Test_createStream(t *testing.T) {
factory
.
f
=
func
(
ctx
context
.
Context
)
(
msgstream
.
MsgStream
,
error
)
{
return
newMockMsgStream
(),
nil
}
_
,
err
:=
createStream
(
factory
,
dmlStreamType
,
[]
string
{
"111"
},
func
(
tsMsgs
[]
msgstream
.
TsMsg
,
hashKeys
[][]
int32
)
(
map
[
int32
]
*
msgstream
.
MsgPack
,
error
)
{
_
,
err
:=
createStream
(
factory
,
[]
string
{
"111"
},
func
(
tsMsgs
[]
msgstream
.
TsMsg
,
hashKeys
[][]
int32
)
(
map
[
int32
]
*
msgstream
.
MsgPack
,
error
)
{
return
nil
,
nil
})
assert
.
NoError
(
t
,
err
)
...
...
@@ -271,7 +271,6 @@ func Test_singleTypeChannelsMgr_createMsgStream(t *testing.T) {
return
channelInfos
{
vchans
:
[]
string
{
"111"
,
"222"
},
pchans
:
[]
string
{
"111"
}},
nil
},
msgStreamFactory
:
factory
,
singleStreamType
:
dmlStreamType
,
repackFunc
:
nil
,
}
_
,
err
:=
m
.
createMsgStream
(
100
)
...
...
@@ -289,7 +288,6 @@ func Test_singleTypeChannelsMgr_createMsgStream(t *testing.T) {
return
channelInfos
{
vchans
:
[]
string
{
"111"
,
"222"
},
pchans
:
[]
string
{
"111"
}},
nil
},
msgStreamFactory
:
factory
,
singleStreamType
:
dmlStreamType
,
repackFunc
:
nil
,
}
stream
,
err
:=
m
.
createMsgStream
(
100
)
...
...
@@ -356,7 +354,6 @@ func Test_singleTypeChannelsMgr_getStream(t *testing.T) {
return
channelInfos
{
vchans
:
[]
string
{
"111"
,
"222"
},
pchans
:
[]
string
{
"111"
}},
nil
},
msgStreamFactory
:
factory
,
singleStreamType
:
dmlStreamType
,
repackFunc
:
nil
,
}
stream
,
err
:=
m
.
getOrCreateStream
(
100
)
...
...
internal/proxy/mock_msgstream_test.go
浏览文件 @
78bc688d
...
...
@@ -58,13 +58,6 @@ func (m *mockMsgStreamFactory) NewTtMsgStream(ctx context.Context) (msgstream.Ms
return
nil
,
errors
.
New
(
"mock"
)
}
func
(
m
*
mockMsgStreamFactory
)
NewQueryMsgStream
(
ctx
context
.
Context
)
(
msgstream
.
MsgStream
,
error
)
{
if
m
.
fQStream
!=
nil
{
return
m
.
fQStream
(
ctx
)
}
return
nil
,
errors
.
New
(
"mock"
)
}
func
newMockMsgStreamFactory
()
*
mockMsgStreamFactory
{
return
&
mockMsgStreamFactory
{}
}
internal/proxy/mock_test.go
浏览文件 @
78bc688d
...
...
@@ -326,10 +326,6 @@ func (factory *simpleMockMsgStreamFactory) NewTtMsgStream(ctx context.Context) (
return
newSimpleMockMsgStream
(),
nil
}
func
(
factory
*
simpleMockMsgStreamFactory
)
NewQueryMsgStream
(
ctx
context
.
Context
)
(
msgstream
.
MsgStream
,
error
)
{
return
newSimpleMockMsgStream
(),
nil
}
func
(
factory
*
simpleMockMsgStreamFactory
)
NewMsgStreamDisposer
(
ctx
context
.
Context
)
func
([]
string
,
string
)
error
{
return
nil
}
...
...
internal/util/dependency/factory.go
浏览文件 @
78bc688d
...
...
@@ -144,10 +144,6 @@ func (f *DefaultFactory) NewTtMsgStream(ctx context.Context) (msgstream.MsgStrea
return
f
.
msgStreamFactory
.
NewTtMsgStream
(
ctx
)
}
func
(
f
*
DefaultFactory
)
NewQueryMsgStream
(
ctx
context
.
Context
)
(
msgstream
.
MsgStream
,
error
)
{
return
f
.
msgStreamFactory
.
NewQueryMsgStream
(
ctx
)
}
func
(
f
*
DefaultFactory
)
NewMsgStreamDisposer
(
ctx
context
.
Context
)
func
([]
string
,
string
)
error
{
return
f
.
msgStreamFactory
.
NewMsgStreamDisposer
(
ctx
)
}
...
...
pkg/mq/msgstream/common_mq_factory.go
浏览文件 @
78bc688d
...
...
@@ -40,16 +40,6 @@ func (f *CommonFactory) NewTtMsgStream(ctx context.Context) (ms MsgStream, err e
return
NewMqTtMsgStream
(
ctx
,
f
.
ReceiveBufSize
,
f
.
MQBufSize
,
cli
,
f
.
DispatcherFactory
.
NewUnmarshalDispatcher
())
}
// NewQueryMsgStream is used to generate a new QueryMsgstream object
func
(
f
*
CommonFactory
)
NewQueryMsgStream
(
ctx
context
.
Context
)
(
ms
MsgStream
,
err
error
)
{
defer
wrapError
(
&
err
,
"NewQueryMsgStream"
)
cli
,
err
:=
f
.
Newer
()
if
err
!=
nil
{
return
nil
,
err
}
return
NewMqMsgStream
(
ctx
,
f
.
ReceiveBufSize
,
f
.
MQBufSize
,
cli
,
f
.
DispatcherFactory
.
NewUnmarshalDispatcher
())
}
// NewMsgStreamDisposer returns a function that can be used to dispose of a message stream.
// The returned function takes a slice of channel names and a subscription name, and
// disposes of the message stream associated with those arguments.
...
...
pkg/mq/msgstream/factory_test.go
浏览文件 @
78bc688d
...
...
@@ -82,9 +82,6 @@ func testFactoryCommonOperation(t *testing.T, f Factory) {
_
,
err
=
f
.
NewTtMsgStream
(
ctx
)
assert
.
NoError
(
t
,
err
)
_
,
err
=
f
.
NewQueryMsgStream
(
ctx
)
assert
.
NoError
(
t
,
err
)
err
=
f
.
NewMsgStreamDisposer
(
ctx
)([]
string
{
"hello"
},
"xx"
)
assert
.
NoError
(
t
,
err
)
}
pkg/mq/msgstream/mq_factory.go
浏览文件 @
78bc688d
...
...
@@ -110,11 +110,6 @@ func (f *PmsFactory) getAuthentication() (pulsar.Authentication, error) {
return
auth
,
nil
}
// NewQueryMsgStream is used to generate a new QueryMsgstream object
func
(
f
*
PmsFactory
)
NewQueryMsgStream
(
ctx
context
.
Context
)
(
MsgStream
,
error
)
{
return
f
.
NewMsgStream
(
ctx
)
}
func
(
f
*
PmsFactory
)
NewMsgStreamDisposer
(
ctx
context
.
Context
)
func
([]
string
,
string
)
error
{
return
func
(
channels
[]
string
,
subname
string
)
error
{
// try to delete the old subscription
...
...
@@ -166,10 +161,6 @@ func (f *KmsFactory) NewTtMsgStream(ctx context.Context) (MsgStream, error) {
return
NewMqTtMsgStream
(
ctx
,
f
.
ReceiveBufSize
,
f
.
MQBufSize
,
kafkaClient
,
f
.
dispatcherFactory
.
NewUnmarshalDispatcher
())
}
func
(
f
*
KmsFactory
)
NewQueryMsgStream
(
ctx
context
.
Context
)
(
MsgStream
,
error
)
{
return
f
.
NewMsgStream
(
ctx
)
}
func
(
f
*
KmsFactory
)
NewMsgStreamDisposer
(
ctx
context
.
Context
)
func
([]
string
,
string
)
error
{
return
func
(
channels
[]
string
,
subname
string
)
error
{
msgstream
,
err
:=
f
.
NewMsgStream
(
ctx
)
...
...
pkg/mq/msgstream/mq_factory_test.go
浏览文件 @
78bc688d
...
...
@@ -34,9 +34,6 @@ func TestPmsFactory(t *testing.T) {
_
,
err
=
pmsFactory
.
NewTtMsgStream
(
ctx
)
assert
.
NoError
(
t
,
err
)
_
,
err
=
pmsFactory
.
NewQueryMsgStream
(
ctx
)
assert
.
NoError
(
t
,
err
)
err
=
pmsFactory
.
NewMsgStreamDisposer
(
ctx
)([]
string
{
"hello"
},
"xx"
)
assert
.
NoError
(
t
,
err
)
}
...
...
@@ -58,9 +55,6 @@ func TestPmsFactoryWithAuth(t *testing.T) {
_
,
err
=
pmsFactory
.
NewTtMsgStream
(
ctx
)
assert
.
NoError
(
t
,
err
)
_
,
err
=
pmsFactory
.
NewQueryMsgStream
(
ctx
)
assert
.
NoError
(
t
,
err
)
Params
.
Save
(
Params
.
PulsarCfg
.
AuthParams
.
Key
,
""
)
pmsFactory
=
NewPmsFactory
(
config
)
...
...
@@ -71,9 +65,6 @@ func TestPmsFactoryWithAuth(t *testing.T) {
_
,
err
=
pmsFactory
.
NewTtMsgStream
(
ctx
)
assert
.
Error
(
t
,
err
)
_
,
err
=
pmsFactory
.
NewQueryMsgStream
(
ctx
)
assert
.
Error
(
t
,
err
)
}
func
TestKafkaFactory
(
t
*
testing
.
T
)
{
...
...
@@ -86,9 +77,6 @@ func TestKafkaFactory(t *testing.T) {
_
,
err
=
kmsFactory
.
NewTtMsgStream
(
ctx
)
assert
.
NoError
(
t
,
err
)
_
,
err
=
kmsFactory
.
NewQueryMsgStream
(
ctx
)
assert
.
NoError
(
t
,
err
)
// err = kmsFactory.NewMsgStreamDisposer(ctx)([]string{"hello"}, "xx")
// assert.NoError(t, err)
}
pkg/mq/msgstream/msgstream.go
浏览文件 @
78bc688d
...
...
@@ -73,6 +73,5 @@ type MsgStream interface {
type
Factory
interface
{
NewMsgStream
(
ctx
context
.
Context
)
(
MsgStream
,
error
)
NewTtMsgStream
(
ctx
context
.
Context
)
(
MsgStream
,
error
)
NewQueryMsgStream
(
ctx
context
.
Context
)
(
MsgStream
,
error
)
NewMsgStreamDisposer
(
ctx
context
.
Context
)
func
([]
string
,
string
)
error
}
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录