Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
33d8e13f
M
milvus
项目概览
milvus
/
milvus
11 个月 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
33d8e13f
编写于
3月 05, 2021
作者:
Y
yukun
提交者:
yefu.chen
3月 05, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Fix load-collection in singlenode
Signed-off-by:
N
yukun
<
kun.yu@zilliz.com
>
上级
74154a11
变更
2
隐藏空白更改
内联
并排
Showing
2 changed file
with
122 addition
and
69 deletion
+122
-69
internal/msgstream/rmqms/rmq_msgstream.go
internal/msgstream/rmqms/rmq_msgstream.go
+85
-35
internal/types/types.go
internal/types/types.go
+37
-34
未找到文件。
internal/msgstream/rmqms/rmq_msgstream.go
浏览文件 @
33d8e13f
...
...
@@ -2,14 +2,13 @@ package rmqms
import
(
"context"
"errors"
"log"
"path/filepath"
"reflect"
"strconv"
"sync"
"errors"
"github.com/gogo/protobuf/proto"
"github.com/zilliztech/milvus-distributed/internal/msgstream/util"
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
...
...
@@ -329,6 +328,7 @@ func (ms *RmqTtMsgStream) AsConsumer(channels []string,
}
consumer
.
MsgNum
=
make
(
chan
int
,
ms
.
rmqBufSize
)
ms
.
consumers
=
append
(
ms
.
consumers
,
*
consumer
)
ms
.
consumerChannels
=
append
(
ms
.
consumerChannels
,
consumer
.
ChannelName
)
ms
.
consumerReflects
=
append
(
ms
.
consumerReflects
,
reflect
.
SelectCase
{
Dir
:
reflect
.
SelectRecv
,
Chan
:
reflect
.
ValueOf
(
consumer
.
MsgNum
),
...
...
@@ -376,6 +376,9 @@ func (ms *RmqTtMsgStream) bufMsgPackToChannel() {
msgPositions
:=
make
([]
*
msgstream
.
MsgPosition
,
0
)
ms
.
unsolvedMutex
.
Lock
()
for
consumer
,
msgs
:=
range
ms
.
unsolvedBuf
{
if
len
(
msgs
)
==
0
{
continue
}
tempBuffer
:=
make
([]
TsMsg
,
0
)
var
timeTickMsg
TsMsg
for
_
,
v
:=
range
msgs
{
...
...
@@ -479,6 +482,10 @@ func (ms *RmqTtMsgStream) Seek(mp *msgstream.MsgPosition) error {
for
index
,
channel
:=
range
ms
.
consumerChannels
{
if
filepath
.
Base
(
channel
)
==
filepath
.
Base
(
mp
.
ChannelName
)
{
consumer
=
ms
.
consumers
[
index
]
if
len
(
mp
.
MsgID
)
==
0
{
msgID
=
-
1
break
}
seekMsgID
,
err
:=
strconv
.
ParseInt
(
mp
.
MsgID
,
10
,
64
)
if
err
!=
nil
{
return
err
...
...
@@ -491,51 +498,94 @@ func (ms *RmqTtMsgStream) Seek(mp *msgstream.MsgPosition) error {
if
err
!=
nil
{
return
err
}
if
msgID
==
-
1
{
return
nil
}
ms
.
unsolvedMutex
.
Lock
()
ms
.
unsolvedBuf
[
consumer
]
=
make
([]
TsMsg
,
0
)
// When rmq seek is called, msgNum can't be used before current msgs all consumed, because
// new msgNum is not generated. So just try to consume msgs
for
{
select
{
case
<-
ms
.
ctx
.
Done
()
:
return
nil
case
num
,
ok
:=
<-
consumer
.
MsgNum
:
if
!
ok
{
return
errors
.
New
(
"consumer closed"
)
rmqMsg
,
err
:=
rocksmq
.
Rmq
.
Consume
(
consumer
.
GroupName
,
consumer
.
ChannelName
,
1
)
if
err
!=
nil
{
log
.
Printf
(
"Failed to consume message in rocksmq, error = %v"
,
err
)
}
if
len
(
rmqMsg
)
==
0
{
break
}
else
{
headerMsg
:=
commonpb
.
MsgHeader
{}
err
:=
proto
.
Unmarshal
(
rmqMsg
[
0
]
.
Payload
,
&
headerMsg
)
if
err
!=
nil
{
log
.
Printf
(
"Failed to unmarshal message header, error = %v"
,
err
)
return
err
}
rmqMsg
,
err
:=
rocksmq
.
Rmq
.
Consume
(
consumer
.
GroupName
,
consumer
.
ChannelName
,
num
)
tsMsg
,
err
:=
ms
.
unmarshal
.
Unmarshal
(
rmqMsg
[
0
]
.
Payload
,
headerMsg
.
Base
.
MsgType
)
if
err
!=
nil
{
log
.
Printf
(
"Failed to
consume message in rocksmq
, error = %v"
,
err
)
continue
log
.
Printf
(
"Failed to
unmarshal tsMsg
, error = %v"
,
err
)
return
err
}
for
j
:=
0
;
j
<
len
(
rmqMsg
);
j
++
{
headerMsg
:=
commonpb
.
MsgHeader
{}
err
:=
proto
.
Unmarshal
(
rmqMsg
[
j
]
.
Payload
,
&
headerMsg
)
if
err
!=
nil
{
log
.
Printf
(
"Failed to unmarshal message header, error = %v"
,
err
)
}
tsMsg
,
err
:=
ms
.
unmarshal
.
Unmarshal
(
rmqMsg
[
j
]
.
Payload
,
headerMsg
.
Base
.
MsgType
)
if
err
!=
nil
{
log
.
Printf
(
"Failed to unmarshal tsMsg, error = %v"
,
err
)
}
if
headerMsg
.
Base
.
MsgType
==
commonpb
.
MsgType_kTimeTick
{
if
tsMsg
.
BeginTs
()
>=
mp
.
Timestamp
{
ms
.
unsolvedMutex
.
Unlock
()
return
nil
}
continue
}
if
tsMsg
.
BeginTs
()
>
mp
.
Timestamp
{
tsMsg
.
SetPosition
(
&
msgstream
.
MsgPosition
{
ChannelName
:
filepath
.
Base
(
consumer
.
ChannelName
),
MsgID
:
strconv
.
Itoa
(
int
(
rmqMsg
[
j
]
.
MsgID
)),
})
ms
.
unsolvedBuf
[
consumer
]
=
append
(
ms
.
unsolvedBuf
[
consumer
],
tsMsg
)
if
headerMsg
.
Base
.
MsgType
==
commonpb
.
MsgType_kTimeTick
{
if
tsMsg
.
BeginTs
()
>=
mp
.
Timestamp
{
ms
.
unsolvedMutex
.
Unlock
()
return
nil
}
continue
}
if
tsMsg
.
BeginTs
()
>
mp
.
Timestamp
{
tsMsg
.
SetPosition
(
&
msgstream
.
MsgPosition
{
ChannelName
:
filepath
.
Base
(
consumer
.
ChannelName
),
MsgID
:
strconv
.
Itoa
(
int
(
rmqMsg
[
0
]
.
MsgID
)),
})
ms
.
unsolvedBuf
[
consumer
]
=
append
(
ms
.
unsolvedBuf
[
consumer
],
tsMsg
)
}
}
}
return
nil
//for {
// select {
// case <-ms.ctx.Done():
// return nil
// case num, ok := <-consumer.MsgNum:
// if !ok {
// return errors.New("consumer closed")
// }
// rmqMsg, err := rocksmq.Rmq.Consume(consumer.GroupName, consumer.ChannelName, num)
// if err != nil {
// log.Printf("Failed to consume message in rocksmq, error = %v", err)
// continue
// }
//
// for j := 0; j < len(rmqMsg); j++ {
// headerMsg := commonpb.MsgHeader{}
// err := proto.Unmarshal(rmqMsg[j].Payload, &headerMsg)
// if err != nil {
// log.Printf("Failed to unmarshal message header, error = %v", err)
// }
// tsMsg, err := ms.unmarshal.Unmarshal(rmqMsg[j].Payload, headerMsg.Base.MsgType)
// if err != nil {
// log.Printf("Failed to unmarshal tsMsg, error = %v", err)
// }
//
// if headerMsg.Base.MsgType == commonpb.MsgType_kTimeTick {
// if tsMsg.BeginTs() >= mp.Timestamp {
// ms.unsolvedMutex.Unlock()
// return nil
// }
// continue
// }
// if tsMsg.BeginTs() > mp.Timestamp {
// tsMsg.SetPosition(&msgstream.MsgPosition{
// ChannelName: filepath.Base(consumer.ChannelName),
// MsgID: strconv.Itoa(int(rmqMsg[j].MsgID)),
// })
// ms.unsolvedBuf[consumer] = append(ms.unsolvedBuf[consumer], tsMsg)
// }
// }
// }
//}
}
func
checkTimeTickMsg
(
msg
map
[
rocksmq
.
Consumer
]
Timestamp
,
...
...
internal/types/types.go
浏览文件 @
33d8e13f
...
...
@@ -25,7 +25,7 @@ type Component interface {
GetStatisticsChannel
(
ctx
context
.
Context
)
(
*
milvuspb
.
StringResponse
,
error
)
}
type
DataNode
Service
interface
{
type
DataNode
interface
{
Component
WatchDmChannels
(
ctx
context
.
Context
,
in
*
datapb
.
WatchDmChannelRequest
)
(
*
commonpb
.
Status
,
error
)
...
...
@@ -51,7 +51,7 @@ type DataService interface {
GetSegmentInfo
(
ctx
context
.
Context
,
req
*
datapb
.
SegmentInfoRequest
)
(
*
datapb
.
SegmentInfoResponse
,
error
)
}
type
IndexNode
Service
interface
{
type
IndexNode
interface
{
Component
TimeTickProvider
...
...
@@ -110,41 +110,44 @@ type MasterService interface {
ShowSegments
(
ctx
context
.
Context
,
in
*
milvuspb
.
ShowSegmentRequest
)
(
*
milvuspb
.
ShowSegmentResponse
,
error
)
}
type
ProxyNode
Service
interface
{
type
ProxyNode
interface
{
Component
InvalidateCollectionMetaCache
(
ctx
context
.
Context
,
request
*
proxypb
.
InvalidateCollMetaCacheRequest
)
(
*
commonpb
.
Status
,
error
)
CreateCollection
(
ctx
context
.
Context
,
request
*
milvuspb
.
CreateCollectionRequest
)
(
*
commonpb
.
Status
,
error
)
DropCollection
(
ctx
context
.
Context
,
request
*
milvuspb
.
DropCollectionRequest
)
(
*
commonpb
.
Status
,
error
)
HasCollection
(
ctx
context
.
Context
,
request
*
milvuspb
.
HasCollectionRequest
)
(
*
milvuspb
.
BoolResponse
,
error
)
LoadCollection
(
ctx
context
.
Context
,
request
*
milvuspb
.
LoadCollectionRequest
)
(
*
commonpb
.
Status
,
error
)
ReleaseCollection
(
ctx
context
.
Context
,
request
*
milvuspb
.
ReleaseCollectionRequest
)
(
*
commonpb
.
Status
,
error
)
DescribeCollection
(
ctx
context
.
Context
,
request
*
milvuspb
.
DescribeCollectionRequest
)
(
*
milvuspb
.
DescribeCollectionResponse
,
error
)
GetCollectionStatistics
(
ctx
context
.
Context
,
request
*
milvuspb
.
CollectionStatsRequest
)
(
*
milvuspb
.
CollectionStatsResponse
,
error
)
ShowCollections
(
ctx
context
.
Context
,
request
*
milvuspb
.
ShowCollectionRequest
)
(
*
milvuspb
.
ShowCollectionResponse
,
error
)
CreatePartition
(
ctx
context
.
Context
,
request
*
milvuspb
.
CreatePartitionRequest
)
(
*
commonpb
.
Status
,
error
)
DropPartition
(
ctx
context
.
Context
,
request
*
milvuspb
.
DropPartitionRequest
)
(
*
commonpb
.
Status
,
error
)
HasPartition
(
ctx
context
.
Context
,
request
*
milvuspb
.
HasPartitionRequest
)
(
*
milvuspb
.
BoolResponse
,
error
)
LoadPartitions
(
ctx
context
.
Context
,
request
*
milvuspb
.
LoadPartitonRequest
)
(
*
commonpb
.
Status
,
error
)
ReleasePartitions
(
ctx
context
.
Context
,
request
*
milvuspb
.
ReleasePartitionRequest
)
(
*
commonpb
.
Status
,
error
)
GetPartitionStatistics
(
ctx
context
.
Context
,
request
*
milvuspb
.
PartitionStatsRequest
)
(
*
milvuspb
.
PartitionStatsResponse
,
error
)
ShowPartitions
(
ctx
context
.
Context
,
request
*
milvuspb
.
ShowPartitionRequest
)
(
*
milvuspb
.
ShowPartitionResponse
,
error
)
CreateIndex
(
ctx
context
.
Context
,
request
*
milvuspb
.
CreateIndexRequest
)
(
*
commonpb
.
Status
,
error
)
DescribeIndex
(
ctx
context
.
Context
,
request
*
milvuspb
.
DescribeIndexRequest
)
(
*
milvuspb
.
DescribeIndexResponse
,
error
)
GetIndexState
(
ctx
context
.
Context
,
request
*
milvuspb
.
IndexStateRequest
)
(
*
milvuspb
.
IndexStateResponse
,
error
)
DropIndex
(
ctx
context
.
Context
,
request
*
milvuspb
.
DropIndexRequest
)
(
*
commonpb
.
Status
,
error
)
Insert
(
ctx
context
.
Context
,
request
*
milvuspb
.
InsertRequest
)
(
*
milvuspb
.
InsertResponse
,
error
)
Search
(
ctx
context
.
Context
,
request
*
milvuspb
.
SearchRequest
)
(
*
milvuspb
.
SearchResults
,
error
)
Flush
(
ctx
context
.
Context
,
request
*
milvuspb
.
FlushRequest
)
(
*
commonpb
.
Status
,
error
)
GetDdChannel
(
ctx
context
.
Context
,
request
*
commonpb
.
Empty
)
(
*
milvuspb
.
StringResponse
,
error
)
GetQuerySegmentInfo
(
ctx
context
.
Context
,
req
*
milvuspb
.
QuerySegmentInfoRequest
)
(
*
milvuspb
.
QuerySegmentInfoResponse
,
error
)
GetPersistentSegmentInfo
(
ctx
context
.
Context
,
req
*
milvuspb
.
PersistentSegmentInfoRequest
)
(
*
milvuspb
.
PersistentSegmentInfoResponse
,
error
)
//TODO: move to milvus service
/*
CreateCollection(ctx context.Context, request *milvuspb.CreateCollectionRequest) (*commonpb.Status, error)
DropCollection(ctx context.Context, request *milvuspb.DropCollectionRequest) (*commonpb.Status, error)
HasCollection(ctx context.Context, request *milvuspb.HasCollectionRequest) (*milvuspb.BoolResponse, error)
LoadCollection(ctx context.Context, request *milvuspb.LoadCollectionRequest) (*commonpb.Status, error)
ReleaseCollection(ctx context.Context, request *milvuspb.ReleaseCollectionRequest) (*commonpb.Status, error)
DescribeCollection(ctx context.Context, request *milvuspb.DescribeCollectionRequest) (*milvuspb.DescribeCollectionResponse, error)
GetCollectionStatistics(ctx context.Context, request *milvuspb.CollectionStatsRequest) (*milvuspb.CollectionStatsResponse, error)
ShowCollections(ctx context.Context, request *milvuspb.ShowCollectionRequest) (*milvuspb.ShowCollectionResponse, error)
CreatePartition(ctx context.Context, request *milvuspb.CreatePartitionRequest) (*commonpb.Status, error)
DropPartition(ctx context.Context, request *milvuspb.DropPartitionRequest) (*commonpb.Status, error)
HasPartition(ctx context.Context, request *milvuspb.HasPartitionRequest) (*milvuspb.BoolResponse, error)
LoadPartitions(ctx context.Context, request *milvuspb.LoadPartitonRequest) (*commonpb.Status, error)
ReleasePartitions(ctx context.Context, request *milvuspb.ReleasePartitionRequest) (*commonpb.Status, error)
GetPartitionStatistics(ctx context.Context, request *milvuspb.PartitionStatsRequest) (*milvuspb.PartitionStatsResponse, error)
ShowPartitions(ctx context.Context, request *milvuspb.ShowPartitionRequest) (*milvuspb.ShowPartitionResponse, error)
CreateIndex(ctx context.Context, request *milvuspb.CreateIndexRequest) (*commonpb.Status, error)
DescribeIndex(ctx context.Context, request *milvuspb.DescribeIndexRequest) (*milvuspb.DescribeIndexResponse, error)
GetIndexState(ctx context.Context, request *milvuspb.IndexStateRequest) (*milvuspb.IndexStateResponse, error)
DropIndex(ctx context.Context, request *milvuspb.DropIndexRequest) (*commonpb.Status, error)
Insert(ctx context.Context, request *milvuspb.InsertRequest) (*milvuspb.InsertResponse, error)
Search(ctx context.Context, request *milvuspb.SearchRequest) (*milvuspb.SearchResults, error)
Flush(ctx context.Context, request *milvuspb.FlushRequest) (*commonpb.Status, error)
GetDdChannel(ctx context.Context, request *commonpb.Empty) (*milvuspb.StringResponse, error)
GetQuerySegmentInfo(ctx context.Context, req *milvuspb.QuerySegmentInfoRequest) (*milvuspb.QuerySegmentInfoResponse, error)
GetPersistentSegmentInfo(ctx context.Context, req *milvuspb.PersistentSegmentInfoRequest) (*milvuspb.PersistentSegmentInfoResponse, error)
*/
}
type
ProxyService
interface
{
...
...
@@ -155,7 +158,7 @@ type ProxyService interface {
InvalidateCollectionMetaCache
(
ctx
context
.
Context
,
request
*
proxypb
.
InvalidateCollMetaCacheRequest
)
(
*
commonpb
.
Status
,
error
)
}
type
QueryNode
Service
interface
{
type
QueryNode
interface
{
Component
TimeTickProvider
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录