Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
ca129d43
M
milvus
项目概览
milvus
/
milvus
11 个月 前同步成功
通知
260
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
ca129d43
编写于
3月 24, 2022
作者:
X
xiyichan
提交者:
GitHub
3月 24, 2022
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Support configurable msgstream (#16131)
Signed-off-by:
N
xiyichan
<
2863768433@qq.com
>
上级
478890a7
变更
31
隐藏空白更改
内联
并排
Showing
31 changed file
with
90 addition
and
214 deletion
+90
-214
cmd/roles/roles.go
cmd/roles/roles.go
+4
-1
docs/developer_guides/chap04_message_stream.md
docs/developer_guides/chap04_message_stream.md
+1
-1
internal/datacoord/server.go
internal/datacoord/server.go
+1
-5
internal/datacoord/server_test.go
internal/datacoord/server_test.go
+1
-6
internal/datanode/data_node.go
internal/datanode/data_node.go
+1
-7
internal/datanode/data_node_test.go
internal/datanode/data_node_test.go
+1
-5
internal/datanode/data_sync_service_test.go
internal/datanode/data_sync_service_test.go
+1
-6
internal/datanode/flow_graph_dmstream_input_node_test.go
internal/datanode/flow_graph_dmstream_input_node_test.go
+6
-5
internal/datanode/flow_graph_insert_buffer_node_test.go
internal/datanode/flow_graph_insert_buffer_node_test.go
+4
-20
internal/mq/msgstream/mq_factory.go
internal/mq/msgstream/mq_factory.go
+11
-13
internal/mq/msgstream/mq_factory_test.go
internal/mq/msgstream/mq_factory_test.go
+10
-32
internal/mq/msgstream/mq_msgstream_test.go
internal/mq/msgstream/mq_msgstream_test.go
+1
-1
internal/mq/msgstream/msgstream.go
internal/mq/msgstream/msgstream.go
+3
-1
internal/proxy/mock_test.go
internal/proxy/mock_test.go
+3
-1
internal/proxy/proxy.go
internal/proxy/proxy.go
+4
-7
internal/querycoord/channel_unsubscribe_test.go
internal/querycoord/channel_unsubscribe_test.go
+1
-5
internal/querycoord/cluster_test.go
internal/querycoord/cluster_test.go
+2
-10
internal/querycoord/query_coord.go
internal/querycoord/query_coord.go
+1
-5
internal/querynode/mock_test.go
internal/querynode/mock_test.go
+5
-12
internal/querynode/query_collection_test.go
internal/querynode/query_collection_test.go
+1
-5
internal/querynode/query_node.go
internal/querynode/query_node.go
+1
-6
internal/querynode/query_node_test.go
internal/querynode/query_node_test.go
+1
-6
internal/querynode/stats_service_test.go
internal/querynode/stats_service_test.go
+2
-10
internal/rootcoord/dml_channels_test.go
internal/rootcoord/dml_channels_test.go
+1
-5
internal/rootcoord/root_coord.go
internal/rootcoord/root_coord.go
+1
-5
internal/rootcoord/root_coord_test.go
internal/rootcoord/root_coord_test.go
+4
-20
internal/rootcoord/timestamp_test.go
internal/rootcoord/timestamp_test.go
+1
-5
internal/rootcoord/timeticksync_test.go
internal/rootcoord/timeticksync_test.go
+1
-5
internal/util/flowgraph/input_node_test.go
internal/util/flowgraph/input_node_test.go
+4
-2
internal/util/flowgraph/node_test.go
internal/util/flowgraph/node_test.go
+4
-2
internal/util/paramtable/component_param.go
internal/util/paramtable/component_param.go
+8
-0
未找到文件。
cmd/roles/roles.go
浏览文件 @
ca129d43
...
...
@@ -67,7 +67,10 @@ func init() {
func
newMsgFactory
(
localMsg
bool
)
msgstream
.
Factory
{
if
localMsg
{
return
msgstream
.
NewRmsFactory
()
if
Params
.
RocksmqEnable
()
{
return
msgstream
.
NewRmsFactory
()
}
return
msgstream
.
NewPmsFactory
()
}
return
msgstream
.
NewPmsFactory
()
}
...
...
docs/developer_guides/chap04_message_stream.md
浏览文件 @
ca129d43
...
...
@@ -213,7 +213,7 @@ type MsgStream interface {
}
type
Factory
interface
{
SetParams
(
params
map
[
string
]
interface
{}
)
error
Init
(
params
*
paramtable
.
ComponentParam
)
error
NewMsgStream
(
ctx
context
.
Context
)
(
MsgStream
,
error
)
NewTtMsgStream
(
ctx
context
.
Context
)
(
MsgStream
,
error
)
NewQueryMsgStream
(
ctx
context
.
Context
)
(
MsgStream
,
error
)
...
...
internal/datacoord/server.go
浏览文件 @
ca129d43
...
...
@@ -259,11 +259,7 @@ func (s *Server) Init() error {
// 4. set server state to Healthy
func
(
s
*
Server
)
Start
()
error
{
var
err
error
m
:=
map
[
string
]
interface
{}{
"PulsarAddress"
:
Params
.
PulsarCfg
.
Address
,
"ReceiveBufSize"
:
1024
,
"PulsarBufSize"
:
1024
}
err
=
s
.
msFactory
.
SetParams
(
m
)
err
=
s
.
msFactory
.
Init
(
&
Params
)
if
err
!=
nil
{
return
err
}
...
...
internal/datacoord/server_test.go
浏览文件 @
ca129d43
...
...
@@ -2320,12 +2320,7 @@ func newTestServer(t *testing.T, receiveCh chan interface{}, opts ...Option) *Se
Params
.
CommonCfg
.
DataCoordTimeTick
=
Params
.
CommonCfg
.
DataCoordTimeTick
+
strconv
.
Itoa
(
rand
.
Int
())
var
err
error
factory
:=
msgstream
.
NewPmsFactory
()
m
:=
map
[
string
]
interface
{}{
"pulsarAddress"
:
Params
.
PulsarCfg
.
Address
,
"receiveBufSize"
:
1024
,
"pulsarBufSize"
:
1024
,
}
err
=
factory
.
SetParams
(
m
)
err
=
factory
.
Init
(
&
Params
)
assert
.
Nil
(
t
,
err
)
etcdCli
,
err
:=
etcd
.
GetEtcdClient
(
&
Params
.
EtcdCfg
)
...
...
internal/datanode/data_node.go
浏览文件 @
ca129d43
...
...
@@ -217,13 +217,7 @@ func (node *DataNode) Init() error {
return
err
}
m
:=
map
[
string
]
interface
{}{
"PulsarAddress"
:
Params
.
PulsarCfg
.
Address
,
"ReceiveBufSize"
:
1024
,
"PulsarBufSize"
:
1024
,
}
if
err
:=
node
.
msFactory
.
SetParams
(
m
);
err
!=
nil
{
if
err
:=
node
.
msFactory
.
Init
(
&
Params
);
err
!=
nil
{
log
.
Warn
(
"DataNode Init msFactory SetParams failed, use default"
,
zap
.
Error
(
err
))
return
err
...
...
internal/datanode/data_node_test.go
浏览文件 @
ca129d43
...
...
@@ -209,11 +209,7 @@ func TestDataNode(t *testing.T) {
// pulsar produce
msFactory
:=
msgstream
.
NewPmsFactory
()
m
:=
map
[
string
]
interface
{}{
"pulsarAddress"
:
Params
.
PulsarCfg
.
Address
,
"receiveBufSize"
:
1024
,
"pulsarBufSize"
:
1024
}
err
=
msFactory
.
SetParams
(
m
)
err
=
msFactory
.
Init
(
&
Params
)
assert
.
NoError
(
t
,
err
)
insertStream
,
err
:=
msFactory
.
NewMsgStream
(
node1
.
ctx
)
assert
.
NoError
(
t
,
err
)
...
...
internal/datanode/data_sync_service_test.go
浏览文件 @
ca129d43
...
...
@@ -181,7 +181,6 @@ func TestDataSyncService_Start(t *testing.T) {
defer
cancel
()
// init data node
pulsarURL
:=
Params
.
PulsarCfg
.
Address
Factory
:=
&
MetaFactory
{}
collMeta
:=
Factory
.
GetCollectionMeta
(
UniqueID
(
0
),
"coll1"
)
...
...
@@ -196,11 +195,7 @@ func TestDataSyncService_Start(t *testing.T) {
allocFactory
:=
NewAllocatorFactory
(
1
)
msFactory
:=
msgstream
.
NewPmsFactory
()
m
:=
map
[
string
]
interface
{}{
"pulsarAddress"
:
pulsarURL
,
"receiveBufSize"
:
1024
,
"pulsarBufSize"
:
1024
}
err
=
msFactory
.
SetParams
(
m
)
err
=
msFactory
.
Init
(
&
Params
)
assert
.
Nil
(
t
,
err
)
insertChannelName
:=
"data_sync_service_test_dml"
...
...
internal/datanode/flow_graph_dmstream_input_node_test.go
浏览文件 @
ca129d43
...
...
@@ -21,6 +21,8 @@ import (
"errors"
"testing"
"github.com/milvus-io/milvus/internal/util/paramtable"
"github.com/milvus-io/milvus/internal/mq/msgstream"
"github.com/milvus-io/milvus/internal/mq/msgstream/mqwrapper"
"github.com/milvus-io/milvus/internal/proto/internalpb"
...
...
@@ -28,17 +30,16 @@ import (
)
type
mockMsgStreamFactory
struct
{
SetParamsReturnNil
bool
InitReturnNil
bool
NewMsgStreamNoError
bool
}
var
_
msgstream
.
Factory
=
&
mockMsgStreamFactory
{}
func
(
mm
*
mockMsgStreamFactory
)
SetParams
(
params
map
[
string
]
interface
{}
)
error
{
if
!
mm
.
SetParams
ReturnNil
{
return
errors
.
New
(
"
Set Params
Error"
)
func
(
mm
*
mockMsgStreamFactory
)
Init
(
params
*
paramtable
.
ComponentParam
)
error
{
if
!
mm
.
Init
ReturnNil
{
return
errors
.
New
(
"
Init
Error"
)
}
return
nil
}
...
...
internal/datanode/flow_graph_insert_buffer_node_test.go
浏览文件 @
ca129d43
...
...
@@ -78,11 +78,7 @@ func TestFlowGraphInsertBufferNodeCreate(t *testing.T) {
require
.
NoError
(
t
,
err
)
msFactory
:=
msgstream
.
NewPmsFactory
()
m
:=
map
[
string
]
interface
{}{
"receiveBufSize"
:
1024
,
"pulsarAddress"
:
Params
.
PulsarCfg
.
Address
,
"pulsarBufSize"
:
1024
}
err
=
msFactory
.
SetParams
(
m
)
err
=
msFactory
.
Init
(
&
Params
)
assert
.
Nil
(
t
,
err
)
fm
:=
NewRendezvousFlushManager
(
&
allocator
{},
cm
,
replica
,
func
(
*
segmentFlushPack
)
{},
emptyFlushAndDropFunc
)
...
...
@@ -168,11 +164,7 @@ func TestFlowGraphInsertBufferNode_Operate(t *testing.T) {
require
.
NoError
(
t
,
err
)
msFactory
:=
msgstream
.
NewPmsFactory
()
m
:=
map
[
string
]
interface
{}{
"receiveBufSize"
:
1024
,
"pulsarAddress"
:
Params
.
PulsarCfg
.
Address
,
"pulsarBufSize"
:
1024
}
err
=
msFactory
.
SetParams
(
m
)
err
=
msFactory
.
Init
(
&
Params
)
assert
.
Nil
(
t
,
err
)
fm
:=
NewRendezvousFlushManager
(
NewAllocatorFactory
(),
cm
,
replica
,
func
(
*
segmentFlushPack
)
{},
emptyFlushAndDropFunc
)
...
...
@@ -372,11 +364,7 @@ func TestFlowGraphInsertBufferNode_AutoFlush(t *testing.T) {
colRep
.
metaService
=
newMetaService
(
mockRootCoord
,
collMeta
.
ID
)
msFactory
:=
msgstream
.
NewPmsFactory
()
m
:=
map
[
string
]
interface
{}{
"receiveBufSize"
:
1024
,
"pulsarAddress"
:
Params
.
PulsarCfg
.
Address
,
"pulsarBufSize"
:
1024
}
err
=
msFactory
.
SetParams
(
m
)
err
=
msFactory
.
Init
(
&
Params
)
assert
.
Nil
(
t
,
err
)
flushPacks
:=
[]
*
segmentFlushPack
{}
...
...
@@ -649,11 +637,7 @@ func TestInsertBufferNode_bufferInsertMsg(t *testing.T) {
require
.
NoError
(
t
,
err
)
msFactory
:=
msgstream
.
NewPmsFactory
()
m
:=
map
[
string
]
interface
{}{
"receiveBufSize"
:
1024
,
"pulsarAddress"
:
Params
.
PulsarCfg
.
Address
,
"pulsarBufSize"
:
1024
}
err
=
msFactory
.
SetParams
(
m
)
err
=
msFactory
.
Init
(
&
Params
)
assert
.
Nil
(
t
,
err
)
fm
:=
NewRendezvousFlushManager
(
&
allocator
{},
cm
,
replica
,
func
(
*
segmentFlushPack
)
{},
emptyFlushAndDropFunc
)
...
...
internal/mq/msgstream/mq_factory.go
浏览文件 @
ca129d43
...
...
@@ -19,12 +19,13 @@ package msgstream
import
(
"context"
"github.com/milvus-io/milvus/internal/util/paramtable"
rmqimplserver
"github.com/milvus-io/milvus/internal/mq/mqimpl/rocksmq/server"
"github.com/apache/pulsar-client-go/pulsar"
puslarmqwrapper
"github.com/milvus-io/milvus/internal/mq/msgstream/mqwrapper/pulsar"
rmqwrapper
"github.com/milvus-io/milvus/internal/mq/msgstream/mqwrapper/rmq"
"github.com/mitchellh/mapstructure"
)
// PmsFactory is a pulsar msgstream factory that implemented Factory interface(msgstream.go)
...
...
@@ -36,12 +37,11 @@ type PmsFactory struct {
PulsarBufSize
int64
}
// SetParams is used to set parameters for PmsFactory
func
(
f
*
PmsFactory
)
SetParams
(
params
map
[
string
]
interface
{})
error
{
err
:=
mapstructure
.
Decode
(
params
,
f
)
if
err
!=
nil
{
return
err
}
// Init is used to set parameters for PmsFactory
func
(
f
*
PmsFactory
)
Init
(
params
*
paramtable
.
ComponentParam
)
error
{
f
.
PulsarBufSize
=
1024
f
.
ReceiveBufSize
=
1024
f
.
PulsarAddress
=
params
.
PulsarCfg
.
Address
return
nil
}
...
...
@@ -86,12 +86,10 @@ type RmsFactory struct {
RmqBufSize
int64
}
// SetParams is used to set parameters for RmsFactory
func
(
f
*
RmsFactory
)
SetParams
(
params
map
[
string
]
interface
{})
error
{
err
:=
mapstructure
.
Decode
(
params
,
f
)
if
err
!=
nil
{
return
err
}
// Init is used to set parameters for RmsFactory
func
(
f
*
RmsFactory
)
Init
(
params
*
paramtable
.
ComponentParam
)
error
{
f
.
RmqBufSize
=
1024
f
.
ReceiveBufSize
=
1024
return
nil
}
...
...
internal/mq/msgstream/mq_factory_test.go
浏览文件 @
ca129d43
...
...
@@ -27,13 +27,7 @@ import (
func
TestPmsFactory
(
t
*
testing
.
T
)
{
pmsFactory
:=
NewPmsFactory
()
pulsarAddress
,
_
:=
Params
.
Load
(
"_PulsarAddress"
)
m
:=
map
[
string
]
interface
{}{
"PulsarAddress"
:
pulsarAddress
,
"receiveBufSize"
:
1024
,
"pulsarBufSize"
:
1024
,
}
pmsFactory
.
SetParams
(
m
)
pmsFactory
.
Init
(
&
Params
)
ctx
:=
context
.
Background
()
_
,
err
:=
pmsFactory
.
NewMsgStream
(
ctx
)
...
...
@@ -46,17 +40,10 @@ func TestPmsFactory(t *testing.T) {
assert
.
Nil
(
t
,
err
)
}
func
TestPmsFactory_SetParams
(
t
*
testing
.
T
)
{
pmsFactory
:=
(
*
PmsFactory
)(
nil
)
pulsarAddress
,
_
:=
Params
.
Load
(
"_PulsarAddress"
)
m
:=
map
[
string
]
interface
{}{
"PulsarAddress"
:
pulsarAddress
,
"receiveBufSize"
:
1024
,
"pulsarBufSize"
:
1024
,
}
err
:=
pmsFactory
.
SetParams
(
m
)
assert
.
NotNil
(
t
,
err
)
func
TestPmsFactory_Init
(
t
*
testing
.
T
)
{
rmsFactory
:=
NewRmsFactory
()
err
:=
rmsFactory
.
Init
(
&
Params
)
assert
.
Nil
(
t
,
err
)
}
func
TestRmsFactory
(
t
*
testing
.
T
)
{
...
...
@@ -65,11 +52,7 @@ func TestRmsFactory(t *testing.T) {
rmsFactory
:=
NewRmsFactory
()
m
:=
map
[
string
]
interface
{}{
"ReceiveBufSize"
:
1024
,
"RmqBufSize"
:
1024
,
}
rmsFactory
.
SetParams
(
m
)
rmsFactory
.
Init
(
&
Params
)
ctx
:=
context
.
Background
()
_
,
err
:=
rmsFactory
.
NewMsgStream
(
ctx
)
...
...
@@ -82,13 +65,8 @@ func TestRmsFactory(t *testing.T) {
assert
.
Nil
(
t
,
err
)
}
func
TestRmsFactory_SetParams
(
t
*
testing
.
T
)
{
rmsFactory
:=
(
*
RmsFactory
)(
nil
)
m
:=
map
[
string
]
interface
{}{
"ReceiveBufSize"
:
1024
,
"RmqBufSize"
:
1024
,
}
err
:=
rmsFactory
.
SetParams
(
m
)
assert
.
NotNil
(
t
,
err
)
func
TestRmsFactory_Init
(
t
*
testing
.
T
)
{
rmsFactory
:=
NewRmsFactory
()
err
:=
rmsFactory
.
Init
(
&
Params
)
assert
.
Nil
(
t
,
err
)
}
internal/mq/msgstream/mq_msgstream_test.go
浏览文件 @
ca129d43
...
...
@@ -48,7 +48,7 @@ import (
"github.com/milvus-io/milvus/internal/util/paramtable"
)
var
Params
paramtable
.
BaseTable
var
Params
paramtable
.
ComponentParam
func
TestMain
(
m
*
testing
.
M
)
{
Params
.
Init
()
...
...
internal/mq/msgstream/msgstream.go
浏览文件 @
ca129d43
...
...
@@ -19,6 +19,8 @@ package msgstream
import
(
"context"
"github.com/milvus-io/milvus/internal/util/paramtable"
"github.com/milvus-io/milvus/internal/mq/msgstream/mqwrapper"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/util/typeutil"
...
...
@@ -75,7 +77,7 @@ type MsgStream interface {
// Factory is an interface that can be used to generate a new msgstream object
type
Factory
interface
{
SetParams
(
params
map
[
string
]
interface
{}
)
error
Init
(
params
*
paramtable
.
ComponentParam
)
error
NewMsgStream
(
ctx
context
.
Context
)
(
MsgStream
,
error
)
NewTtMsgStream
(
ctx
context
.
Context
)
(
MsgStream
,
error
)
NewQueryMsgStream
(
ctx
context
.
Context
)
(
MsgStream
,
error
)
...
...
internal/proxy/mock_test.go
浏览文件 @
ca129d43
...
...
@@ -22,6 +22,8 @@ import (
"sync"
"time"
"github.com/milvus-io/milvus/internal/util/paramtable"
"github.com/milvus-io/milvus/internal/mq/msgstream"
"github.com/milvus-io/milvus/internal/mq/msgstream/mqwrapper"
"github.com/milvus-io/milvus/internal/proto/commonpb"
...
...
@@ -373,7 +375,7 @@ func newSimpleMockMsgStream() *simpleMockMsgStream {
type
simpleMockMsgStreamFactory
struct
{
}
func
(
factory
*
simpleMockMsgStreamFactory
)
SetParams
(
params
map
[
string
]
interface
{}
)
error
{
func
(
factory
*
simpleMockMsgStreamFactory
)
Init
(
param
*
paramtable
.
ComponentParam
)
error
{
return
nil
}
...
...
internal/proxy/proxy.go
浏览文件 @
ca129d43
...
...
@@ -182,18 +182,15 @@ func (node *Proxy) Init() error {
log
.
Debug
(
"create query channel for Proxy done"
,
zap
.
String
(
"QueryResultChannel"
,
resp
.
QueryResultChannel
))
}
m
:=
map
[
string
]
interface
{}{
"PulsarAddress"
:
Params
.
PulsarCfg
.
Address
,
"PulsarBufSize"
:
1024
}
log
.
Debug
(
"set parameters for ms factory"
,
zap
.
String
(
"role"
,
typeutil
.
ProxyRole
),
zap
.
Any
(
"parameters"
,
m
))
if
err
:=
node
.
msFactory
.
SetParams
(
m
);
err
!=
nil
{
log
.
Debug
(
"set parameters for ms factory"
,
zap
.
String
(
"role"
,
typeutil
.
ProxyRole
),
zap
.
Any
(
"parameters"
,
Params
.
ServiceParam
))
if
err
:=
node
.
msFactory
.
Init
(
&
Params
);
err
!=
nil
{
log
.
Warn
(
"failed to set parameters for ms factory"
,
zap
.
Error
(
err
),
zap
.
String
(
"role"
,
typeutil
.
ProxyRole
),
zap
.
Any
(
"parameters"
,
m
))
zap
.
Any
(
"parameters"
,
Params
.
ServicePara
m
))
return
err
}
log
.
Debug
(
"set parameters for ms factory done"
,
zap
.
String
(
"role"
,
typeutil
.
ProxyRole
),
zap
.
Any
(
"parameters"
,
m
))
log
.
Debug
(
"set parameters for ms factory done"
,
zap
.
String
(
"role"
,
typeutil
.
ProxyRole
),
zap
.
Any
(
"parameters"
,
Params
.
ServicePara
m
))
log
.
Debug
(
"create id allocator"
,
zap
.
String
(
"role"
,
typeutil
.
ProxyRole
),
zap
.
Int64
(
"ProxyID"
,
Params
.
ProxyCfg
.
ProxyID
))
idAllocator
,
err
:=
allocator
.
NewIDAllocator
(
node
.
ctx
,
node
.
rootCoord
,
Params
.
ProxyCfg
.
ProxyID
)
...
...
internal/querycoord/channel_unsubscribe_test.go
浏览文件 @
ca129d43
...
...
@@ -96,11 +96,7 @@ func Test_HandleChannelUnsubscribeLoop(t *testing.T) {
defer
etcdCli
.
Close
()
kv
:=
etcdkv
.
NewEtcdKV
(
etcdCli
,
Params
.
EtcdCfg
.
MetaRootPath
)
factory
:=
msgstream
.
NewPmsFactory
()
m
:=
map
[
string
]
interface
{}{
"PulsarAddress"
:
Params
.
PulsarCfg
.
Address
,
"ReceiveBufSize"
:
1024
,
"PulsarBufSize"
:
1024
}
factory
.
SetParams
(
m
)
factory
.
Init
(
&
Params
)
handler
,
err
:=
newChannelUnsubscribeHandler
(
baseCtx
,
kv
,
factory
)
assert
.
Nil
(
t
,
err
)
...
...
internal/querycoord/cluster_test.go
浏览文件 @
ca129d43
...
...
@@ -457,11 +457,7 @@ func TestGrpcRequest(t *testing.T) {
clusterSession
.
Init
(
typeutil
.
QueryCoordRole
,
Params
.
QueryCoordCfg
.
Address
,
true
,
false
)
clusterSession
.
Register
()
factory
:=
msgstream
.
NewPmsFactory
()
m
:=
map
[
string
]
interface
{}{
"PulsarAddress"
:
Params
.
PulsarCfg
.
Address
,
"ReceiveBufSize"
:
1024
,
"PulsarBufSize"
:
1024
}
err
=
factory
.
SetParams
(
m
)
err
=
factory
.
Init
(
&
Params
)
assert
.
Nil
(
t
,
err
)
idAllocator
:=
func
()
(
UniqueID
,
error
)
{
return
0
,
nil
...
...
@@ -652,11 +648,7 @@ func TestSetNodeState(t *testing.T) {
clusterSession
.
Init
(
typeutil
.
QueryCoordRole
,
Params
.
QueryCoordCfg
.
Address
,
true
,
false
)
clusterSession
.
Register
()
factory
:=
msgstream
.
NewPmsFactory
()
m
:=
map
[
string
]
interface
{}{
"PulsarAddress"
:
Params
.
PulsarCfg
.
Address
,
"ReceiveBufSize"
:
1024
,
"PulsarBufSize"
:
1024
}
err
=
factory
.
SetParams
(
m
)
err
=
factory
.
Init
(
&
Params
)
assert
.
Nil
(
t
,
err
)
idAllocator
:=
func
()
(
UniqueID
,
error
)
{
return
0
,
nil
...
...
internal/querycoord/query_coord.go
浏览文件 @
ca129d43
...
...
@@ -222,11 +222,7 @@ func (qc *QueryCoord) Init() error {
// Start function starts the goroutines to watch the meta and node updates
func
(
qc
*
QueryCoord
)
Start
()
error
{
m
:=
map
[
string
]
interface
{}{
"PulsarAddress"
:
Params
.
PulsarCfg
.
Address
,
"ReceiveBufSize"
:
1024
,
"PulsarBufSize"
:
1024
}
err
:=
qc
.
msFactory
.
SetParams
(
m
)
err
:=
qc
.
msFactory
.
Init
(
&
Params
)
if
err
!=
nil
{
return
err
}
...
...
internal/querynode/mock_test.go
浏览文件 @
ca129d43
...
...
@@ -24,6 +24,8 @@ import (
"math/rand"
"strconv"
"github.com/milvus-io/milvus/internal/util/paramtable"
"github.com/milvus-io/milvus/internal/util/indexcgowrapper"
"github.com/golang/protobuf/proto"
...
...
@@ -623,13 +625,8 @@ func genEtcdKV() (*etcdkv.EtcdKV, error) {
func
genFactory
()
(
msgstream
.
Factory
,
error
)
{
const
receiveBufSize
=
1024
pulsarURL
:=
Params
.
PulsarCfg
.
Address
msFactory
:=
msgstream
.
NewPmsFactory
()
m
:=
map
[
string
]
interface
{}{
"receiveBufSize"
:
receiveBufSize
,
"pulsarAddress"
:
pulsarURL
,
"pulsarBufSize"
:
1024
}
err
:=
msFactory
.
SetParams
(
m
)
err
:=
msFactory
.
Init
(
&
Params
)
if
err
!=
nil
{
return
nil
,
err
}
...
...
@@ -640,11 +637,7 @@ func genInvalidFactory() (msgstream.Factory, error) {
const
receiveBufSize
=
1024
msFactory
:=
msgstream
.
NewPmsFactory
()
m
:=
map
[
string
]
interface
{}{
"receiveBufSize"
:
receiveBufSize
,
"pulsarAddress"
:
""
,
"pulsarBufSize"
:
1024
}
err
:=
msFactory
.
SetParams
(
m
)
err
:=
msFactory
.
Init
(
&
Params
)
if
err
!=
nil
{
return
nil
,
err
}
...
...
@@ -1835,7 +1828,7 @@ type mockMsgStreamFactory struct {
var
_
msgstream
.
Factory
=
&
mockMsgStreamFactory
{}
func
(
mm
*
mockMsgStreamFactory
)
SetParams
(
params
map
[
string
]
interface
{}
)
error
{
func
(
mm
*
mockMsgStreamFactory
)
Init
(
params
*
paramtable
.
ComponentParam
)
error
{
return
nil
}
...
...
internal/querynode/query_collection_test.go
浏览文件 @
ca129d43
...
...
@@ -125,12 +125,8 @@ func updateTSafe(queryCollection *queryCollection, timestamp Timestamp) error {
func
TestQueryCollection_withoutVChannel
(
t
*
testing
.
T
)
{
ctx
:=
context
.
Background
()
m
:=
map
[
string
]
interface
{}{
"PulsarAddress"
:
Params
.
PulsarCfg
.
Address
,
"ReceiveBufSize"
:
1024
,
"PulsarBufSize"
:
1024
}
factory
:=
msgstream
.
NewPmsFactory
()
err
:=
factory
.
SetParams
(
m
)
err
:=
factory
.
Init
(
&
Params
)
assert
.
Nil
(
t
,
err
)
etcdCli
,
err
:=
etcd
.
GetEtcdClient
(
&
Params
.
EtcdCfg
)
assert
.
Nil
(
t
,
err
)
...
...
internal/querynode/query_node.go
浏览文件 @
ca129d43
...
...
@@ -331,12 +331,7 @@ func (node *QueryNode) Init() error {
// Start mainly start QueryNode's query service.
func
(
node
*
QueryNode
)
Start
()
error
{
var
err
error
m
:=
map
[
string
]
interface
{}{
"PulsarAddress"
:
Params
.
PulsarCfg
.
Address
,
"ReceiveBufSize"
:
1024
,
"PulsarBufSize"
:
1024
}
err
=
node
.
msFactory
.
SetParams
(
m
)
err
:=
node
.
msFactory
.
Init
(
&
Params
)
if
err
!=
nil
{
return
err
}
...
...
internal/querynode/query_node_test.go
浏览文件 @
ca129d43
...
...
@@ -227,13 +227,8 @@ func makeNewChannelNames(names []string, suffix string) []string {
func
newMessageStreamFactory
()
(
msgstream
.
Factory
,
error
)
{
const
receiveBufSize
=
1024
pulsarURL
:=
Params
.
PulsarCfg
.
Address
msFactory
:=
msgstream
.
NewPmsFactory
()
m
:=
map
[
string
]
interface
{}{
"receiveBufSize"
:
receiveBufSize
,
"pulsarAddress"
:
pulsarURL
,
"pulsarBufSize"
:
1024
}
err
:=
msFactory
.
SetParams
(
m
)
err
:=
msFactory
.
Init
(
&
Params
)
return
msFactory
,
err
}
...
...
internal/querynode/stats_service_test.go
浏览文件 @
ca129d43
...
...
@@ -30,11 +30,7 @@ func TestStatsService_start(t *testing.T) {
initTestMeta
(
t
,
node
,
0
,
0
)
msFactory
:=
msgstream
.
NewPmsFactory
()
m
:=
map
[
string
]
interface
{}{
"PulsarAddress"
:
Params
.
PulsarCfg
.
Address
,
"ReceiveBufSize"
:
1024
,
"PulsarBufSize"
:
1024
}
msFactory
.
SetParams
(
m
)
msFactory
.
Init
(
&
Params
)
node
.
statsService
=
newStatsService
(
node
.
queryNodeLoopCtx
,
node
.
historical
.
replica
,
msFactory
)
node
.
statsService
.
start
()
node
.
Stop
()
...
...
@@ -53,11 +49,7 @@ func TestSegmentManagement_sendSegmentStatistic(t *testing.T) {
producerChannels
:=
[]
string
{
Params
.
CommonCfg
.
QueryNodeStats
}
msFactory
:=
msgstream
.
NewPmsFactory
()
m
:=
map
[
string
]
interface
{}{
"receiveBufSize"
:
receiveBufSize
,
"pulsarAddress"
:
Params
.
PulsarCfg
.
Address
,
"pulsarBufSize"
:
1024
}
err
=
msFactory
.
SetParams
(
m
)
err
=
msFactory
.
Init
(
&
Params
)
assert
.
Nil
(
t
,
err
)
statsStream
,
err
:=
msFactory
.
NewMsgStream
(
node
.
queryNodeLoopCtx
)
...
...
internal/rootcoord/dml_channels_test.go
浏览文件 @
ca129d43
...
...
@@ -41,11 +41,7 @@ func TestDmlChannels(t *testing.T) {
factory
:=
msgstream
.
NewPmsFactory
()
Params
.
Init
()
m
:=
map
[
string
]
interface
{}{
"pulsarAddress"
:
Params
.
PulsarCfg
.
Address
,
"receiveBufSize"
:
1024
,
"pulsarBufSize"
:
1024
}
err
:=
factory
.
SetParams
(
m
)
err
:=
factory
.
Init
(
&
Params
)
assert
.
Nil
(
t
,
err
)
dml
:=
newDmlChannels
(
ctx
,
factory
,
dmlChanPrefix
,
totalDmlChannelNum
)
...
...
internal/rootcoord/root_coord.go
浏览文件 @
ca129d43
...
...
@@ -1055,11 +1055,7 @@ func (c *Core) Init() error {
return
tsoAllocator
.
GetLastSavedTime
()
}
m
:=
map
[
string
]
interface
{}{
"PulsarAddress"
:
Params
.
PulsarCfg
.
Address
,
"ReceiveBufSize"
:
1024
,
"PulsarBufSize"
:
1024
}
if
initError
=
c
.
msFactory
.
SetParams
(
m
);
initError
!=
nil
{
if
initError
=
c
.
msFactory
.
Init
(
&
Params
);
initError
!=
nil
{
return
}
...
...
internal/rootcoord/root_coord_test.go
浏览文件 @
ca129d43
...
...
@@ -628,11 +628,7 @@ func TestRootCoord(t *testing.T) {
tmpFactory
:=
msgstream
.
NewPmsFactory
()
m
:=
map
[
string
]
interface
{}{
"pulsarAddress"
:
Params
.
PulsarCfg
.
Address
,
"receiveBufSize"
:
1024
,
"pulsarBufSize"
:
1024
}
err
=
tmpFactory
.
SetParams
(
m
)
err
=
tmpFactory
.
Init
(
&
Params
)
assert
.
Nil
(
t
,
err
)
timeTickStream
,
_
:=
tmpFactory
.
NewMsgStream
(
ctx
)
...
...
@@ -2420,11 +2416,7 @@ func TestRootCoord2(t *testing.T) {
err
=
core
.
Register
()
assert
.
Nil
(
t
,
err
)
m
:=
map
[
string
]
interface
{}{
"receiveBufSize"
:
1024
,
"pulsarAddress"
:
Params
.
PulsarCfg
.
Address
,
"pulsarBufSize"
:
1024
}
err
=
msFactory
.
SetParams
(
m
)
err
=
msFactory
.
Init
(
&
Params
)
assert
.
Nil
(
t
,
err
)
timeTickStream
,
_
:=
msFactory
.
NewMsgStream
(
ctx
)
...
...
@@ -2708,11 +2700,7 @@ func TestCheckFlushedSegments(t *testing.T) {
err
=
core
.
Register
()
assert
.
Nil
(
t
,
err
)
m
:=
map
[
string
]
interface
{}{
"receiveBufSize"
:
1024
,
"pulsarAddress"
:
Params
.
PulsarCfg
.
Address
,
"pulsarBufSize"
:
1024
}
err
=
msFactory
.
SetParams
(
m
)
err
=
msFactory
.
Init
(
&
Params
)
assert
.
Nil
(
t
,
err
)
timeTickStream
,
_
:=
msFactory
.
NewMsgStream
(
ctx
)
...
...
@@ -2875,11 +2863,7 @@ func TestRootCoord_CheckZeroShardsNum(t *testing.T) {
err
=
core
.
Register
()
assert
.
Nil
(
t
,
err
)
m
:=
map
[
string
]
interface
{}{
"receiveBufSize"
:
1024
,
"pulsarAddress"
:
Params
.
PulsarCfg
.
Address
,
"pulsarBufSize"
:
1024
}
err
=
msFactory
.
SetParams
(
m
)
err
=
msFactory
.
Init
(
&
Params
)
assert
.
Nil
(
t
,
err
)
timeTickStream
,
_
:=
msFactory
.
NewMsgStream
(
ctx
)
...
...
internal/rootcoord/timestamp_test.go
浏览文件 @
ca129d43
...
...
@@ -120,11 +120,7 @@ func BenchmarkAllocTimestamp(b *testing.B) {
err
=
core
.
Start
()
assert
.
Nil
(
b
,
err
)
m
:=
map
[
string
]
interface
{}{
"receiveBufSize"
:
1024
,
"pulsarAddress"
:
Params
.
PulsarCfg
.
Address
,
"pulsarBufSize"
:
1024
}
err
=
msFactory
.
SetParams
(
m
)
err
=
msFactory
.
Init
(
&
Params
)
assert
.
Nil
(
b
,
err
)
b
.
ResetTimer
()
...
...
internal/rootcoord/timeticksync_test.go
浏览文件 @
ca129d43
...
...
@@ -33,11 +33,7 @@ func TestTimetickSync(t *testing.T) {
sourceID
:=
int64
(
100
)
factory
:=
msgstream
.
NewPmsFactory
()
m
:=
map
[
string
]
interface
{}{
"pulsarAddress"
:
Params
.
PulsarCfg
.
Address
,
"receiveBufSize"
:
1024
,
"pulsarBufSize"
:
1024
}
err
:=
factory
.
SetParams
(
m
)
err
:=
factory
.
Init
(
&
Params
)
assert
.
Nil
(
t
,
err
)
//chanMap := map[typeutil.UniqueID][]string{
...
...
internal/util/flowgraph/input_node_test.go
浏览文件 @
ca129d43
...
...
@@ -21,6 +21,8 @@ import (
"os"
"testing"
"github.com/milvus-io/milvus/internal/util/paramtable"
"github.com/milvus-io/milvus/internal/mq/msgstream"
"github.com/stretchr/testify/assert"
)
...
...
@@ -28,8 +30,8 @@ import (
func
TestInputNode
(
t
*
testing
.
T
)
{
os
.
Setenv
(
"ROCKSMQ_PATH"
,
"/tmp/MilvusTest/FlowGraph/TestInputNode"
)
msFactory
:=
msgstream
.
NewRmsFactory
()
m
:=
map
[
string
]
interface
{}{}
err
:=
msFactory
.
SetParams
(
m
)
var
Params
paramtable
.
ComponentParam
err
:=
msFactory
.
Init
(
&
Params
)
assert
.
Nil
(
t
,
err
)
msgStream
,
_
:=
msFactory
.
NewMsgStream
(
context
.
TODO
())
...
...
internal/util/flowgraph/node_test.go
浏览文件 @
ca129d43
...
...
@@ -24,6 +24,8 @@ import (
"testing"
"time"
"github.com/milvus-io/milvus/internal/util/paramtable"
"github.com/milvus-io/milvus/internal/mq/msgstream"
"github.com/milvus-io/milvus/internal/proto/commonpb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
...
...
@@ -57,8 +59,8 @@ func generateMsgPack() msgstream.MsgPack {
func
TestNodeCtx_Start
(
t
*
testing
.
T
)
{
os
.
Setenv
(
"ROCKSMQ_PATH"
,
"/tmp/MilvusTest/FlowGraph/TestNodeStart"
)
msFactory
:=
msgstream
.
NewRmsFactory
()
m
:=
map
[
string
]
interface
{}{}
err
:=
msFactory
.
SetParams
(
m
)
var
Params
paramtable
.
ComponentParam
err
:=
msFactory
.
Init
(
&
Params
)
assert
.
Nil
(
t
,
err
)
msgStream
,
_
:=
msFactory
.
NewMsgStream
(
context
.
TODO
())
...
...
internal/util/paramtable/component_param.go
浏览文件 @
ca129d43
...
...
@@ -75,6 +75,14 @@ func (p *ComponentParam) SetLogConfig(role string) {
p
.
BaseTable
.
SetLogConfig
()
}
func
(
p
*
ComponentParam
)
RocksmqEnable
()
bool
{
return
p
.
RocksmqCfg
.
Path
!=
""
}
func
(
p
*
ComponentParam
)
PulsarEnable
()
bool
{
return
p
.
PulsarCfg
.
Address
!=
""
}
///////////////////////////////////////////////////////////////////////////////
// --- common ---
type
commonConfig
struct
{
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录