Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
59a3d83f
M
milvus
项目概览
milvus
/
milvus
12 个月 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
59a3d83f
编写于
5月 27, 2021
作者:
D
dragondriver
提交者:
zhenshan.cao
6月 15, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Replace insertChannelsMap with channelsMgr (#5453)
Signed-off-by:
N
dragondriver
<
jiquan.long@zilliz.com
>
上级
607a8ffa
变更
5
隐藏空白更改
内联
并排
Showing
5 changed file
with
52 addition
and
87 deletion
+52
-87
internal/proxynode/channels_mgr.go
internal/proxynode/channels_mgr.go
+36
-0
internal/proxynode/channels_mgr_test.go
internal/proxynode/channels_mgr_test.go
+0
-37
internal/proxynode/impl.go
internal/proxynode/impl.go
+2
-0
internal/proxynode/proxy_node.go
internal/proxynode/proxy_node.go
+7
-0
internal/proxynode/task.go
internal/proxynode/task.go
+7
-50
未找到文件。
internal/proxynode/channels_mgr.go
浏览文件 @
59a3d83f
...
...
@@ -3,6 +3,7 @@ package proxynode
import
(
"context"
"fmt"
"math/rand"
"runtime"
"sort"
"sync"
...
...
@@ -64,6 +65,41 @@ type masterService interface {
GetChannels
(
collectionID
UniqueID
)
(
map
[
vChan
]
pChan
,
error
)
}
type
mockMaster
struct
{
collectionID2Channels
map
[
UniqueID
]
map
[
vChan
]
pChan
}
func
newMockMaster
()
*
mockMaster
{
return
&
mockMaster
{
collectionID2Channels
:
make
(
map
[
UniqueID
]
map
[
vChan
]
pChan
),
}
}
func
genUniqueStr
()
string
{
l
:=
rand
.
Uint64
()
%
100
+
1
b
:=
make
([]
byte
,
l
)
if
_
,
err
:=
rand
.
Read
(
b
);
err
!=
nil
{
return
""
}
return
fmt
.
Sprintf
(
"%X"
,
b
)
}
func
(
m
*
mockMaster
)
GetChannels
(
collectionID
UniqueID
)
(
map
[
vChan
]
pChan
,
error
)
{
channels
,
ok
:=
m
.
collectionID2Channels
[
collectionID
]
if
ok
{
return
channels
,
nil
}
channels
=
make
(
map
[
vChan
]
pChan
)
l
:=
rand
.
Uint64
()
%
10
+
1
for
i
:=
0
;
uint64
(
i
)
<
l
;
i
++
{
channels
[
genUniqueStr
()]
=
genUniqueStr
()
}
m
.
collectionID2Channels
[
collectionID
]
=
channels
return
channels
,
nil
}
type
channelsMgrImpl
struct
{
collectionID2VIDs
map
[
UniqueID
][]
int
// id are sorted
collMtx
sync
.
RWMutex
...
...
internal/proxynode/channels_mgr_test.go
浏览文件 @
59a3d83f
package
proxynode
import
(
"fmt"
"math/rand"
"testing"
"github.com/milvus-io/milvus/internal/msgstream"
...
...
@@ -24,41 +22,6 @@ func TestNaiveUniqueIntGenerator_get(t *testing.T) {
}
}
type
mockMaster
struct
{
collectionID2Channels
map
[
UniqueID
]
map
[
vChan
]
pChan
}
func
newMockMaster
()
*
mockMaster
{
return
&
mockMaster
{
collectionID2Channels
:
make
(
map
[
UniqueID
]
map
[
vChan
]
pChan
),
}
}
func
genUniqueStr
()
string
{
l
:=
rand
.
Uint64
()
%
100
+
1
b
:=
make
([]
byte
,
l
)
if
_
,
err
:=
rand
.
Read
(
b
);
err
!=
nil
{
return
""
}
return
fmt
.
Sprintf
(
"%X"
,
b
)
}
func
(
m
*
mockMaster
)
GetChannels
(
collectionID
UniqueID
)
(
map
[
vChan
]
pChan
,
error
)
{
channels
,
ok
:=
m
.
collectionID2Channels
[
collectionID
]
if
ok
{
return
channels
,
nil
}
channels
=
make
(
map
[
vChan
]
pChan
)
l
:=
rand
.
Uint64
()
%
10
+
1
for
i
:=
0
;
uint64
(
i
)
<
l
;
i
++
{
channels
[
genUniqueStr
()]
=
genUniqueStr
()
}
m
.
collectionID2Channels
[
collectionID
]
=
channels
return
channels
,
nil
}
func
TestChannelsMgrImpl_getChannels
(
t
*
testing
.
T
)
{
master
:=
newMockMaster
()
factory
:=
msgstream
.
NewSimpleMsgStreamFactory
()
...
...
internal/proxynode/impl.go
浏览文件 @
59a3d83f
...
...
@@ -143,6 +143,7 @@ func (node *ProxyNode) DropCollection(ctx context.Context, request *milvuspb.Dro
Condition
:
NewTaskCondition
(
ctx
),
DropCollectionRequest
:
request
,
masterService
:
node
.
masterService
,
chMgr
:
node
.
chMgr
,
}
err
:=
node
.
sched
.
DdQueue
.
Enqueue
(
dct
)
...
...
@@ -1062,6 +1063,7 @@ func (node *ProxyNode) Insert(ctx context.Context, request *milvuspb.InsertReque
},
rowIDAllocator
:
node
.
idAllocator
,
segIDAssigner
:
node
.
segAssigner
,
chMgr
:
node
.
chMgr
,
}
if
len
(
it
.
PartitionName
)
<=
0
{
it
.
PartitionName
=
Params
.
DefaultPartitionName
...
...
internal/proxynode/proxy_node.go
浏览文件 @
59a3d83f
...
...
@@ -54,6 +54,8 @@ type ProxyNode struct {
proxyService
types
.
ProxyService
queryService
types
.
QueryService
chMgr
channelsMgr
sched
*
TaskScheduler
tick
*
timeTick
...
...
@@ -224,6 +226,11 @@ func (node *ProxyNode) Init() error {
node
.
segAssigner
=
segAssigner
node
.
segAssigner
.
PeerID
=
Params
.
ProxyID
// TODO(dragondriver): use real master service instance
mockMasterIns
:=
newMockMaster
()
chMgr
:=
newChannelsMgr
(
mockMasterIns
,
node
.
msFactory
)
node
.
chMgr
=
chMgr
node
.
sched
,
err
=
NewTaskScheduler
(
node
.
ctx
,
node
.
idAllocator
,
node
.
tsoAllocator
,
node
.
msFactory
)
if
err
!=
nil
{
return
err
...
...
internal/proxynode/task.go
浏览文件 @
59a3d83f
...
...
@@ -106,6 +106,7 @@ type InsertTask struct {
result
*
milvuspb
.
InsertResponse
rowIDAllocator
*
allocator
.
IDAllocator
segIDAssigner
*
SegIDAssigner
chMgr
channelsMgr
}
func
(
it
*
InsertTask
)
TraceCtx
()
context
.
Context
{
...
...
@@ -684,30 +685,14 @@ func (it *InsertTask) Execute(ctx context.Context) error {
msgPack
.
Msgs
[
0
]
=
tsMsg
stream
,
err
:=
globalInsertChannelsMap
.
GetInsertMsg
Stream
(
collID
)
stream
,
err
:=
it
.
chMgr
.
getDML
Stream
(
collID
)
if
err
!=
nil
{
resp
,
_
:=
it
.
dataService
.
GetInsertChannels
(
ctx
,
&
datapb
.
GetInsertChannelsRequest
{
Base
:
&
commonpb
.
MsgBase
{
MsgType
:
commonpb
.
MsgType_Insert
,
// todo
MsgID
:
it
.
Base
.
MsgID
,
// todo
Timestamp
:
0
,
// todo
SourceID
:
Params
.
ProxyID
,
},
DbID
:
0
,
// todo
CollectionID
:
collID
,
})
if
resp
==
nil
{
return
errors
.
New
(
"get insert channels resp is nil"
)
}
if
resp
.
Status
.
ErrorCode
!=
commonpb
.
ErrorCode_Success
{
return
errors
.
New
(
resp
.
Status
.
Reason
)
}
err
=
globalInsertChannelsMap
.
CreateInsertMsgStream
(
collID
,
resp
.
Values
)
err
=
it
.
chMgr
.
createDMLMsgStream
(
collID
)
if
err
!=
nil
{
return
err
}
}
stream
,
err
=
globalInsertChannelsMap
.
GetInsertMsg
Stream
(
collID
)
stream
,
err
=
it
.
chMgr
.
getDML
Stream
(
collID
)
if
err
!=
nil
{
it
.
result
.
Status
.
ErrorCode
=
commonpb
.
ErrorCode_UnexpectedError
it
.
result
.
Status
.
Reason
=
err
.
Error
()
...
...
@@ -849,36 +834,7 @@ func (cct *CreateCollectionTask) PreExecute(ctx context.Context) error {
func
(
cct
*
CreateCollectionTask
)
Execute
(
ctx
context
.
Context
)
error
{
var
err
error
cct
.
result
,
err
=
cct
.
masterService
.
CreateCollection
(
ctx
,
cct
.
CreateCollectionRequest
)
if
err
!=
nil
{
return
err
}
if
cct
.
result
.
ErrorCode
==
commonpb
.
ErrorCode_Success
{
collID
,
err
:=
globalMetaCache
.
GetCollectionID
(
ctx
,
cct
.
CollectionName
)
if
err
!=
nil
{
return
err
}
resp
,
_
:=
cct
.
dataServiceClient
.
GetInsertChannels
(
ctx
,
&
datapb
.
GetInsertChannelsRequest
{
Base
:
&
commonpb
.
MsgBase
{
MsgType
:
commonpb
.
MsgType_Insert
,
// todo
MsgID
:
cct
.
Base
.
MsgID
,
// todo
Timestamp
:
0
,
// todo
SourceID
:
Params
.
ProxyID
,
},
DbID
:
0
,
// todo
CollectionID
:
collID
,
})
if
resp
==
nil
{
return
errors
.
New
(
"get insert channels resp is nil"
)
}
if
resp
.
Status
.
ErrorCode
!=
commonpb
.
ErrorCode_Success
{
return
errors
.
New
(
resp
.
Status
.
Reason
)
}
err
=
globalInsertChannelsMap
.
CreateInsertMsgStream
(
collID
,
resp
.
Values
)
if
err
!=
nil
{
return
err
}
}
return
nil
return
err
}
func
(
cct
*
CreateCollectionTask
)
PostExecute
(
ctx
context
.
Context
)
error
{
...
...
@@ -891,6 +847,7 @@ type DropCollectionTask struct {
ctx
context
.
Context
masterService
types
.
MasterService
result
*
commonpb
.
Status
chMgr
channelsMgr
}
func
(
dct
*
DropCollectionTask
)
TraceCtx
()
context
.
Context
{
...
...
@@ -951,7 +908,7 @@ func (dct *DropCollectionTask) Execute(ctx context.Context) error {
return
err
}
err
=
globalInsertChannelsMap
.
CloseInsertMsg
Stream
(
collID
)
err
=
dct
.
chMgr
.
removeDML
Stream
(
collID
)
if
err
!=
nil
{
return
err
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录