Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
e786ad6a
M
milvus
项目概览
milvus
/
milvus
大约 1 年 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
e786ad6a
编写于
2月 22, 2021
作者:
Y
yukun
提交者:
yefu.chen
2月 22, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Fix inconsistency of produce msg and consume msg with rmq
Signed-off-by:
N
yukun
<
kun.yu@zilliz.com
>
上级
b07a371f
变更
7
隐藏空白更改
内联
并排
Showing
7 changed file
with
69 addition
and
43 deletion
+69
-43
internal/msgstream/rmqms/rmq_msgstream.go
internal/msgstream/rmqms/rmq_msgstream.go
+1
-33
internal/proxynode/paramtable.go
internal/proxynode/paramtable.go
+1
-0
internal/querynode/load_service.go
internal/querynode/load_service.go
+6
-1
internal/querynode/load_service_test.go
internal/querynode/load_service_test.go
+1
-1
internal/querynode/segment_loader.go
internal/querynode/segment_loader.go
+9
-7
internal/util/rocksmq/rocksmq.go
internal/util/rocksmq/rocksmq.go
+2
-1
internal/util/rocksmq/rocksmq_test.go
internal/util/rocksmq/rocksmq_test.go
+49
-0
未找到文件。
internal/msgstream/rmqms/rmq_msgstream.go
浏览文件 @
e786ad6a
...
@@ -144,7 +144,7 @@ func (ms *RmqMsgStream) Produce(pack *msgstream.MsgPack) error {
...
@@ -144,7 +144,7 @@ func (ms *RmqMsgStream) Produce(pack *msgstream.MsgPack) error {
channelID
:=
searchResult
.
ResultChannelID
channelID
:=
searchResult
.
ResultChannelID
channelIDInt
,
_
:=
strconv
.
ParseInt
(
channelID
,
10
,
64
)
channelIDInt
,
_
:=
strconv
.
ParseInt
(
channelID
,
10
,
64
)
if
channelIDInt
>=
int64
(
len
(
ms
.
producers
))
{
if
channelIDInt
>=
int64
(
len
(
ms
.
producers
))
{
return
errors
.
New
(
"Failed to produce
pulsar
msg to unKnow channel"
)
return
errors
.
New
(
"Failed to produce
rmq
msg to unKnow channel"
)
}
}
bucketValues
[
index
]
=
int32
(
channelIDInt
)
bucketValues
[
index
]
=
int32
(
channelIDInt
)
continue
continue
...
@@ -178,38 +178,6 @@ func (ms *RmqMsgStream) Produce(pack *msgstream.MsgPack) error {
...
@@ -178,38 +178,6 @@ func (ms *RmqMsgStream) Produce(pack *msgstream.MsgPack) error {
if
err
!=
nil
{
if
err
!=
nil
{
return
err
return
err
}
}
//
//msg := &pulsar.ProducerMessage{Payload: mb}
//var child opentracing.Span
if
v
.
Msgs
[
i
]
.
Type
()
==
commonpb
.
MsgType_kInsert
||
v
.
Msgs
[
i
]
.
Type
()
==
commonpb
.
MsgType_kSearch
||
v
.
Msgs
[
i
]
.
Type
()
==
commonpb
.
MsgType_kSearchResult
{
//tracer := opentracing.GlobalTracer()
//ctx := v.Msgs[i].GetMsgContext()
//if ctx == nil {
// ctx = context.Background()
//}
//
//if parent := opentracing.SpanFromContext(ctx); parent != nil {
// child = tracer.StartSpan("start send pulsar msg",
// opentracing.FollowsFrom(parent.Context()))
//} else {
// child = tracer.StartSpan("start send pulsar msg")
//}
//child.SetTag("hash keys", v.Msgs[i].HashKeys())
//child.SetTag("start time", v.Msgs[i].BeginTs())
//child.SetTag("end time", v.Msgs[i].EndTs())
//child.SetTag("msg type", v.Msgs[i].Type())
//msg.Properties = make(map[string]string)
//err = tracer.Inject(child.Context(), opentracing.TextMap, &propertiesReaderWriter{msg.Properties})
//if err != nil {
// child.LogFields(oplog.Error(err))
// child.Finish()
// return err
//}
//child.LogFields(oplog.String("inject success", "inject success"))
}
m
,
err
:=
msgstream
.
ConvertToByteArray
(
mb
)
m
,
err
:=
msgstream
.
ConvertToByteArray
(
mb
)
if
err
!=
nil
{
if
err
!=
nil
{
...
...
internal/proxynode/paramtable.go
浏览文件 @
e786ad6a
...
@@ -177,6 +177,7 @@ func (pt *ParamTable) initQueryNodeIDList() []UniqueID {
...
@@ -177,6 +177,7 @@ func (pt *ParamTable) initQueryNodeIDList() []UniqueID {
}
}
ret
=
append
(
ret
,
UniqueID
(
v
))
ret
=
append
(
ret
,
UniqueID
(
v
))
}
}
pt
.
QueryNodeIDList
=
ret
return
ret
return
ret
}
}
...
...
internal/querynode/load_service.go
浏览文件 @
e786ad6a
...
@@ -114,7 +114,12 @@ func (s *loadService) loadSegmentInternal(collectionID UniqueID, partitionID Uni
...
@@ -114,7 +114,12 @@ func (s *loadService) loadSegmentInternal(collectionID UniqueID, partitionID Uni
return
err
return
err
}
}
targetFields
:=
s
.
segLoader
.
getTargetFields
(
paths
,
srcFieldIDs
,
fieldIDs
)
//fmt.Println("srcFieldIDs in internal:", srcFieldIDs)
//fmt.Println("dstFieldIDs in internal:", fieldIDs)
targetFields
,
err
:=
s
.
segLoader
.
checkTargetFields
(
paths
,
srcFieldIDs
,
fieldIDs
)
if
err
!=
nil
{
return
err
}
err
=
s
.
segLoader
.
loadSegmentFieldsData
(
segment
,
targetFields
)
err
=
s
.
segLoader
.
loadSegmentFieldsData
(
segment
,
targetFields
)
if
err
!=
nil
{
if
err
!=
nil
{
return
err
return
err
...
...
internal/querynode/load_service_test.go
浏览文件 @
e786ad6a
...
@@ -1142,7 +1142,7 @@ func TestSegmentLoad_Search_Vector(t *testing.T) {
...
@@ -1142,7 +1142,7 @@ func TestSegmentLoad_Search_Vector(t *testing.T) {
paths
,
srcFieldIDs
,
err
:=
generateInsertBinLog
(
collectionID
,
partitionID
,
segmentID
,
keyPrefix
)
paths
,
srcFieldIDs
,
err
:=
generateInsertBinLog
(
collectionID
,
partitionID
,
segmentID
,
keyPrefix
)
assert
.
NoError
(
t
,
err
)
assert
.
NoError
(
t
,
err
)
fieldsMap
:=
node
.
loadService
.
segLoader
.
get
TargetFields
(
paths
,
srcFieldIDs
,
fieldIDs
)
fieldsMap
,
_
:=
node
.
loadService
.
segLoader
.
check
TargetFields
(
paths
,
srcFieldIDs
,
fieldIDs
)
assert
.
Equal
(
t
,
len
(
fieldsMap
),
2
)
assert
.
Equal
(
t
,
len
(
fieldsMap
),
2
)
segment
,
err
:=
node
.
replica
.
getSegmentByID
(
segmentID
)
segment
,
err
:=
node
.
replica
.
getSegmentByID
(
segmentID
)
...
...
internal/querynode/segment_loader.go
浏览文件 @
e786ad6a
...
@@ -8,6 +8,7 @@ import (
...
@@ -8,6 +8,7 @@ import (
"github.com/zilliztech/milvus-distributed/internal/kv"
"github.com/zilliztech/milvus-distributed/internal/kv"
minioKV
"github.com/zilliztech/milvus-distributed/internal/kv/minio"
minioKV
"github.com/zilliztech/milvus-distributed/internal/kv/minio"
"github.com/zilliztech/milvus-distributed/internal/msgstream"
"github.com/zilliztech/milvus-distributed/internal/msgstream"
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
"github.com/zilliztech/milvus-distributed/internal/proto/datapb"
"github.com/zilliztech/milvus-distributed/internal/proto/datapb"
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb2"
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb2"
"github.com/zilliztech/milvus-distributed/internal/storage"
"github.com/zilliztech/milvus-distributed/internal/storage"
...
@@ -53,11 +54,11 @@ func (loader *segmentLoader) getInsertBinlogPaths(segmentID UniqueID) ([]*intern
...
@@ -53,11 +54,11 @@ func (loader *segmentLoader) getInsertBinlogPaths(segmentID UniqueID) ([]*intern
}
}
pathResponse
,
err
:=
loader
.
dataClient
.
GetInsertBinlogPaths
(
insertBinlogPathRequest
)
pathResponse
,
err
:=
loader
.
dataClient
.
GetInsertBinlogPaths
(
insertBinlogPathRequest
)
if
err
!=
nil
{
if
err
!=
nil
||
pathResponse
.
Status
.
ErrorCode
!=
commonpb
.
ErrorCode_SUCCESS
{
return
nil
,
nil
,
err
return
nil
,
nil
,
err
}
}
if
len
(
pathResponse
.
FieldIDs
)
!=
len
(
pathResponse
.
Paths
)
{
if
len
(
pathResponse
.
FieldIDs
)
!=
len
(
pathResponse
.
Paths
)
||
len
(
pathResponse
.
FieldIDs
)
<=
0
{
return
nil
,
nil
,
errors
.
New
(
"illegal InsertBinlogPathsResponse"
)
return
nil
,
nil
,
errors
.
New
(
"illegal InsertBinlogPathsResponse"
)
}
}
...
@@ -82,7 +83,7 @@ func (loader *segmentLoader) filterOutVectorFields(fieldIDs []int64, vectorField
...
@@ -82,7 +83,7 @@ func (loader *segmentLoader) filterOutVectorFields(fieldIDs []int64, vectorField
return
targetFields
return
targetFields
}
}
func
(
loader
*
segmentLoader
)
getTargetFields
(
paths
[]
*
internalpb2
.
StringList
,
srcFieldIDS
[]
int64
,
dstFields
[]
int64
)
map
[
int64
]
*
internalpb2
.
StringList
{
func
(
loader
*
segmentLoader
)
checkTargetFields
(
paths
[]
*
internalpb2
.
StringList
,
srcFieldIDs
[]
int64
,
dstFieldIDs
[]
int64
)
(
map
[
int64
]
*
internalpb2
.
StringList
,
error
)
{
targetFields
:=
make
(
map
[
int64
]
*
internalpb2
.
StringList
)
targetFields
:=
make
(
map
[
int64
]
*
internalpb2
.
StringList
)
containsFunc
:=
func
(
s
[]
int64
,
e
int64
)
bool
{
containsFunc
:=
func
(
s
[]
int64
,
e
int64
)
bool
{
...
@@ -94,13 +95,14 @@ func (loader *segmentLoader) getTargetFields(paths []*internalpb2.StringList, sr
...
@@ -94,13 +95,14 @@ func (loader *segmentLoader) getTargetFields(paths []*internalpb2.StringList, sr
return
false
return
false
}
}
for
i
,
fieldID
:=
range
srcFieldIDS
{
for
i
,
fieldID
:=
range
dstFieldIDs
{
if
containsFunc
(
dstField
s
,
fieldID
)
{
if
!
containsFunc
(
srcFieldID
s
,
fieldID
)
{
targetFields
[
fieldID
]
=
paths
[
i
]
return
nil
,
errors
.
New
(
"uncompleted fields"
)
}
}
targetFields
[
fieldID
]
=
paths
[
i
]
}
}
return
targetFields
return
targetFields
,
nil
}
}
func
(
loader
*
segmentLoader
)
loadSegmentFieldsData
(
segment
*
Segment
,
targetFields
map
[
int64
]
*
internalpb2
.
StringList
)
error
{
func
(
loader
*
segmentLoader
)
loadSegmentFieldsData
(
segment
*
Segment
,
targetFields
map
[
int64
]
*
internalpb2
.
StringList
)
error
{
...
...
internal/util/rocksmq/rocksmq.go
浏览文件 @
e786ad6a
...
@@ -286,7 +286,8 @@ func (rmq *RocksMQ) Consume(groupName string, channelName string, n int) ([]Cons
...
@@ -286,7 +286,8 @@ func (rmq *RocksMQ) Consume(groupName string, channelName string, n int) ([]Cons
if
iter
.
Seek
([]
byte
(
dataKey
));
currentID
!=
DefaultMessageID
&&
iter
.
Valid
()
{
if
iter
.
Seek
([]
byte
(
dataKey
));
currentID
!=
DefaultMessageID
&&
iter
.
Valid
()
{
iter
.
Next
()
iter
.
Next
()
}
else
{
}
else
{
iter
.
SeekToFirst
()
newKey
:=
fixChanName
+
"/"
iter
.
Seek
([]
byte
(
newKey
))
}
}
offset
:=
0
offset
:=
0
...
...
internal/util/rocksmq/rocksmq_test.go
浏览文件 @
e786ad6a
...
@@ -269,3 +269,52 @@ func TestRocksMQ_Throughout(t *testing.T) {
...
@@ -269,3 +269,52 @@ func TestRocksMQ_Throughout(t *testing.T) {
cDuration
:=
ct1
-
ct0
cDuration
:=
ct1
-
ct0
log
.
Printf
(
"Total consume %d item, cost %v ms, throughout %v / s"
,
entityNum
,
cDuration
,
int64
(
entityNum
)
*
1000
/
cDuration
)
log
.
Printf
(
"Total consume %d item, cost %v ms, throughout %v / s"
,
entityNum
,
cDuration
,
int64
(
entityNum
)
*
1000
/
cDuration
)
}
}
func
TestRocksMQ_MultiChan
(
t
*
testing
.
T
)
{
etcdAddr
:=
os
.
Getenv
(
"ETCD_ADDRESS"
)
if
etcdAddr
==
""
{
etcdAddr
=
"localhost:2379"
}
cli
,
err
:=
clientv3
.
New
(
clientv3
.
Config
{
Endpoints
:
[]
string
{
etcdAddr
}})
assert
.
Nil
(
t
,
err
)
etcdKV
:=
etcdkv
.
NewEtcdKV
(
cli
,
"/etcd/test/root"
)
defer
etcdKV
.
Close
()
idAllocator
:=
NewGlobalIDAllocator
(
"dummy"
,
etcdKV
)
_
=
idAllocator
.
Initialize
()
name
:=
"/tmp/rocksmq_multichan"
defer
os
.
RemoveAll
(
name
)
rmq
,
err
:=
NewRocksMQ
(
name
,
idAllocator
)
assert
.
Nil
(
t
,
err
)
channelName0
:=
"chan01"
channelName1
:=
"chan11"
err
=
rmq
.
CreateChannel
(
channelName0
)
assert
.
Nil
(
t
,
err
)
defer
rmq
.
DestroyChannel
(
channelName0
)
err
=
rmq
.
CreateChannel
(
channelName1
)
assert
.
Nil
(
t
,
err
)
defer
rmq
.
DestroyChannel
(
channelName1
)
assert
.
Nil
(
t
,
err
)
loopNum
:=
10
for
i
:=
0
;
i
<
loopNum
;
i
++
{
msg0
:=
"for_chann0_"
+
strconv
.
Itoa
(
i
)
msg1
:=
"for_chann1_"
+
strconv
.
Itoa
(
i
)
pMsg0
:=
ProducerMessage
{
payload
:
[]
byte
(
msg0
)}
pMsg1
:=
ProducerMessage
{
payload
:
[]
byte
(
msg1
)}
err
=
rmq
.
Produce
(
channelName0
,
[]
ProducerMessage
{
pMsg0
})
assert
.
Nil
(
t
,
err
)
err
=
rmq
.
Produce
(
channelName1
,
[]
ProducerMessage
{
pMsg1
})
assert
.
Nil
(
t
,
err
)
}
groupName
:=
"test_group"
_
=
rmq
.
DestroyConsumerGroup
(
groupName
,
channelName1
)
_
,
err
=
rmq
.
CreateConsumerGroup
(
groupName
,
channelName1
)
assert
.
Nil
(
t
,
err
)
cMsgs
,
err
:=
rmq
.
Consume
(
groupName
,
channelName1
,
1
)
assert
.
Nil
(
t
,
err
)
assert
.
Equal
(
t
,
len
(
cMsgs
),
1
)
assert
.
Equal
(
t
,
string
(
cMsgs
[
0
]
.
Payload
),
"for_chann1_"
+
strconv
.
Itoa
(
0
))
}
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录