Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
5698bf42
M
milvus
项目概览
milvus
/
milvus
11 个月 前同步成功
通知
260
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
前往新版Gitcode,体验更适合开发者的 AI 搜索 >>
未验证
提交
5698bf42
编写于
6月 02, 2022
作者:
J
jaime
提交者:
GitHub
6月 02, 2022
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Add sasl configurations for kafka (#17323)
Signed-off-by:
N
yun.zhang
<
yun.zhang@zilliz.com
>
上级
e88ffb8a
变更
5
隐藏空白更改
内联
并排
Showing
5 changed file
with
61 addition
and
12 deletion
+61
-12
configs/milvus.yaml
configs/milvus.yaml
+2
-0
internal/mq/msgstream/mq_factory.go
internal/mq/msgstream/mq_factory.go
+4
-4
internal/mq/msgstream/mqwrapper/kafka/kafka_client.go
internal/mq/msgstream/mqwrapper/kafka/kafka_client.go
+28
-6
internal/mq/msgstream/mqwrapper/kafka/kafka_client_test.go
internal/mq/msgstream/mqwrapper/kafka/kafka_client_test.go
+13
-0
internal/util/paramtable/service_param.go
internal/util/paramtable/service_param.go
+14
-2
未找到文件。
configs/milvus.yaml
浏览文件 @
5698bf42
...
...
@@ -75,6 +75,8 @@ pulsar:
# If you want to enable kafka, needs to comment the pulsar configs
#kafka:
# brokerList: localhost1:9092,localhost2:9092,localhost3:9092
# saslUsername: username
# saslPassword: password
rocksmq
:
# please adjust in embedded Milvus: /tmp/milvus/rdb_data
...
...
internal/mq/msgstream/mq_factory.go
浏览文件 @
5698bf42
...
...
@@ -124,17 +124,17 @@ func NewRmsFactory(path string) *RmsFactory {
type
KmsFactory
struct
{
dispatcherFactory
ProtoUDFactory
KafkaAddress
strin
g
config
*
paramtable
.
KafkaConfi
g
ReceiveBufSize
int64
}
func
(
f
*
KmsFactory
)
NewMsgStream
(
ctx
context
.
Context
)
(
MsgStream
,
error
)
{
kafkaClient
:=
kafkawrapper
.
NewKafkaClientInstance
(
f
.
KafkaAddress
)
kafkaClient
:=
kafkawrapper
.
NewKafkaClientInstance
WithConfig
(
f
.
config
)
return
NewMqMsgStream
(
ctx
,
f
.
ReceiveBufSize
,
-
1
,
kafkaClient
,
f
.
dispatcherFactory
.
NewUnmarshalDispatcher
())
}
func
(
f
*
KmsFactory
)
NewTtMsgStream
(
ctx
context
.
Context
)
(
MsgStream
,
error
)
{
kafkaClient
:=
kafkawrapper
.
NewKafkaClientInstance
(
f
.
KafkaAddress
)
kafkaClient
:=
kafkawrapper
.
NewKafkaClientInstance
WithConfig
(
f
.
config
)
return
NewMqTtMsgStream
(
ctx
,
f
.
ReceiveBufSize
,
-
1
,
kafkaClient
,
f
.
dispatcherFactory
.
NewUnmarshalDispatcher
())
}
...
...
@@ -146,7 +146,7 @@ func NewKmsFactory(config *paramtable.KafkaConfig) Factory {
f
:=
&
KmsFactory
{
dispatcherFactory
:
ProtoUDFactory
{},
ReceiveBufSize
:
1024
,
KafkaAddress
:
config
.
Address
,
config
:
config
,
}
return
f
}
internal/mq/msgstream/mqwrapper/kafka/kafka_client.go
浏览文件 @
5698bf42
...
...
@@ -4,6 +4,8 @@ import (
"strconv"
"sync"
"github.com/milvus-io/milvus/internal/util/paramtable"
"github.com/confluentinc/confluent-kafka-go/kafka"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/mq/msgstream/mqwrapper"
...
...
@@ -18,18 +20,38 @@ type kafkaClient struct {
basicConfig
kafka
.
ConfigMap
}
func
NewKafkaClientInstance
(
address
string
)
*
kafkaClient
{
config
:=
kafka
.
ConfigMap
{
"bootstrap.servers"
:
address
,
"socket.timeout.ms"
:
300000
,
"socket.max.fails"
:
3
,
//"receive.message.max.bytes": 10485760,
func
getBasicConfig
(
address
string
)
kafka
.
ConfigMap
{
return
kafka
.
ConfigMap
{
"bootstrap.servers"
:
address
,
"socket.timeout.ms"
:
300000
,
"socket.max.fails"
:
3
,
"api.version.request"
:
true
,
}
}
func
NewKafkaClientInstance
(
address
string
)
*
kafkaClient
{
config
:=
getBasicConfig
(
address
)
return
&
kafkaClient
{
basicConfig
:
config
}
}
func
NewKafkaClientInstanceWithConfig
(
config
*
paramtable
.
KafkaConfig
)
*
kafkaClient
{
kafkaConfig
:=
getBasicConfig
(
config
.
Address
)
if
(
config
.
SaslUsername
==
""
&&
config
.
SaslPassword
!=
""
)
||
(
config
.
SaslUsername
!=
""
&&
config
.
SaslPassword
==
""
)
{
panic
(
"enable security mode need config username and password at the same time!"
)
}
if
config
.
SaslUsername
!=
""
&&
config
.
SaslPassword
!=
""
{
kafkaConfig
.
SetKey
(
"sasl.mechanisms"
,
"PLAIN"
)
kafkaConfig
.
SetKey
(
"security.protocol"
,
"SASL_SSL"
)
kafkaConfig
.
SetKey
(
"sasl.username"
,
config
.
SaslUsername
)
kafkaConfig
.
SetKey
(
"sasl.password"
,
config
.
SaslPassword
)
}
return
&
kafkaClient
{
basicConfig
:
kafkaConfig
}
}
func
cloneKafkaConfig
(
config
kafka
.
ConfigMap
)
*
kafka
.
ConfigMap
{
newConfig
:=
make
(
kafka
.
ConfigMap
)
for
k
,
v
:=
range
config
{
...
...
internal/mq/msgstream/mqwrapper/kafka/kafka_client_test.go
浏览文件 @
5698bf42
...
...
@@ -278,6 +278,19 @@ func TestKafkaClient_MsgSerializAndDeserialize(t *testing.T) {
assert
.
Nil
(
t
,
msgID
)
}
func
TestKafkaClient_NewKafkaClientInstanceWithConfig
(
t
*
testing
.
T
)
{
config1
:=
&
paramtable
.
KafkaConfig
{
Address
:
"addr"
,
SaslPassword
:
"password"
}
assert
.
Panics
(
t
,
func
()
{
NewKafkaClientInstanceWithConfig
(
config1
)
})
config2
:=
&
paramtable
.
KafkaConfig
{
Address
:
"addr"
,
SaslUsername
:
"username"
}
assert
.
Panics
(
t
,
func
()
{
NewKafkaClientInstanceWithConfig
(
config2
)
})
config3
:=
&
paramtable
.
KafkaConfig
{
Address
:
"addr"
,
SaslUsername
:
"username"
,
SaslPassword
:
"password"
}
client
:=
NewKafkaClientInstanceWithConfig
(
config3
)
assert
.
NotNil
(
t
,
client
)
assert
.
NotNil
(
t
,
client
.
basicConfig
)
}
func
createKafkaClient
(
t
*
testing
.
T
)
*
kafkaClient
{
kafkaAddress
,
_
:=
Params
.
Load
(
"_KafkaBrokerList"
)
kc
:=
NewKafkaClientInstance
(
kafkaAddress
)
...
...
internal/util/paramtable/service_param.go
浏览文件 @
5698bf42
...
...
@@ -234,13 +234,17 @@ func (p *PulsarConfig) initMaxMessageSize() {
// --- kafka ---
type
KafkaConfig
struct
{
Base
*
BaseTable
Address
string
Base
*
BaseTable
Address
string
SaslUsername
string
SaslPassword
string
}
func
(
k
*
KafkaConfig
)
init
(
base
*
BaseTable
)
{
k
.
Base
=
base
k
.
initAddress
()
k
.
initSaslUsername
()
k
.
initSaslPassword
()
}
func
(
k
*
KafkaConfig
)
initAddress
()
{
...
...
@@ -251,6 +255,14 @@ func (k *KafkaConfig) initAddress() {
k
.
Address
=
addr
}
func
(
k
*
KafkaConfig
)
initSaslUsername
()
{
k
.
SaslUsername
=
k
.
Base
.
LoadWithDefault
(
"kafka.saslUsername"
,
""
)
}
func
(
k
*
KafkaConfig
)
initSaslPassword
()
{
k
.
SaslPassword
=
k
.
Base
.
LoadWithDefault
(
"kafka.saslPassword"
,
""
)
}
///////////////////////////////////////////////////////////////////////////////
// --- rocksmq ---
type
RocksmqConfig
struct
{
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录