Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
91f5c480
M
milvus
项目概览
milvus
/
milvus
11 个月 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
91f5c480
编写于
3月 30, 2021
作者:
X
XuanYang-cn
提交者:
yefu.chen
3月 30, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Fix datanode goroutine leak
Signed-off-by:
N
XuanYang-cn
<
xuan.yang@zilliz.com
>
上级
721cdb2f
变更
6
隐藏空白更改
内联
并排
Showing
6 changed file
with
35 addition
and
23 deletion
+35
-23
internal/datanode/data_node.go
internal/datanode/data_node.go
+4
-3
internal/datanode/data_node_test.go
internal/datanode/data_node_test.go
+5
-5
internal/datanode/data_sync_service.go
internal/datanode/data_sync_service.go
+2
-2
internal/datanode/flow_graph_dd_node.go
internal/datanode/flow_graph_dd_node.go
+2
-2
internal/datanode/flow_graph_insert_buffer_node.go
internal/datanode/flow_graph_insert_buffer_node.go
+19
-10
internal/datanode/flow_graph_insert_buffer_node_test.go
internal/datanode/flow_graph_insert_buffer_node_test.go
+3
-1
未找到文件。
internal/datanode/data_node.go
浏览文件 @
91f5c480
...
@@ -40,7 +40,7 @@ type DataNode struct {
...
@@ -40,7 +40,7 @@ type DataNode struct {
masterService
types
.
MasterService
masterService
types
.
MasterService
dataService
types
.
DataService
dataService
types
.
DataService
flushChan
chan
*
flushMsg
flushChan
chan
<-
*
flushMsg
replica
Replica
replica
Replica
closer
io
.
Closer
closer
io
.
Closer
...
@@ -135,9 +135,10 @@ func (node *DataNode) Init() error {
...
@@ -135,9 +135,10 @@ func (node *DataNode) Init() error {
var
alloc
allocatorInterface
=
newAllocator
(
node
.
masterService
)
var
alloc
allocatorInterface
=
newAllocator
(
node
.
masterService
)
chanSize
:=
100
chanSize
:=
100
node
.
flushChan
=
make
(
chan
*
flushMsg
,
chanSize
)
flushChan
:=
make
(
chan
*
flushMsg
,
chanSize
)
node
.
flushChan
=
flushChan
node
.
dataSyncService
=
newDataSyncService
(
node
.
ctx
,
node
.
flushChan
,
replica
,
alloc
,
node
.
msFactory
)
node
.
dataSyncService
=
newDataSyncService
(
node
.
ctx
,
flushChan
,
replica
,
alloc
,
node
.
msFactory
)
node
.
dataSyncService
.
init
()
node
.
dataSyncService
.
init
()
node
.
metaService
=
newMetaService
(
node
.
ctx
,
replica
,
node
.
masterService
)
node
.
metaService
=
newMetaService
(
node
.
ctx
,
replica
,
node
.
masterService
)
...
...
internal/datanode/data_node_test.go
浏览文件 @
91f5c480
...
@@ -237,11 +237,11 @@ func (mf *MetaFactory) CollectionMetaFactory(collectionID UniqueID, collectionNa
...
@@ -237,11 +237,11 @@ func (mf *MetaFactory) CollectionMetaFactory(collectionID UniqueID, collectionNa
}
}
collection
:=
etcdpb
.
CollectionMeta
{
collection
:=
etcdpb
.
CollectionMeta
{
ID
:
collectionID
,
ID
:
collectionID
,
Schema
:
&
sch
,
Schema
:
&
sch
,
CreateTime
:
Timestamp
(
1
),
CreateTime
:
Timestamp
(
1
),
SegmentIDs
:
make
([]
UniqueID
,
0
),
SegmentIDs
:
make
([]
UniqueID
,
0
),
Partition
Tags
:
make
([]
string
,
0
)
,
Partition
IDs
:
[]
UniqueID
{
0
}
,
}
}
return
&
collection
return
&
collection
}
}
...
...
internal/datanode/data_sync_service.go
浏览文件 @
91f5c480
...
@@ -17,13 +17,13 @@ import (
...
@@ -17,13 +17,13 @@ import (
type
dataSyncService
struct
{
type
dataSyncService
struct
{
ctx
context
.
Context
ctx
context
.
Context
fg
*
flowgraph
.
TimeTickedFlowGraph
fg
*
flowgraph
.
TimeTickedFlowGraph
flushChan
chan
*
flushMsg
flushChan
<-
chan
*
flushMsg
replica
Replica
replica
Replica
idAllocator
allocatorInterface
idAllocator
allocatorInterface
msFactory
msgstream
.
Factory
msFactory
msgstream
.
Factory
}
}
func
newDataSyncService
(
ctx
context
.
Context
,
flushChan
chan
*
flushMsg
,
func
newDataSyncService
(
ctx
context
.
Context
,
flushChan
<-
chan
*
flushMsg
,
replica
Replica
,
alloc
allocatorInterface
,
factory
msgstream
.
Factory
)
*
dataSyncService
{
replica
Replica
,
alloc
allocatorInterface
,
factory
msgstream
.
Factory
)
*
dataSyncService
{
service
:=
&
dataSyncService
{
service
:=
&
dataSyncService
{
ctx
:
ctx
,
ctx
:
ctx
,
...
...
internal/datanode/flow_graph_dd_node.go
浏览文件 @
91f5c480
...
@@ -29,7 +29,7 @@ type ddNode struct {
...
@@ -29,7 +29,7 @@ type ddNode struct {
ddRecords
*
ddRecords
ddRecords
*
ddRecords
ddBuffer
*
ddBuffer
ddBuffer
*
ddBuffer
flushMap
*
sync
.
Map
flushMap
*
sync
.
Map
inFlushCh
chan
*
flushMsg
inFlushCh
<-
chan
*
flushMsg
kv
kv
.
Base
kv
kv
.
Base
replica
Replica
replica
Replica
...
@@ -429,7 +429,7 @@ func (ddNode *ddNode) dropPartition(msg *msgstream.DropPartitionMsg) {
...
@@ -429,7 +429,7 @@ func (ddNode *ddNode) dropPartition(msg *msgstream.DropPartitionMsg) {
}
}
func
newDDNode
(
ctx
context
.
Context
,
binlogMeta
*
binlogMeta
,
func
newDDNode
(
ctx
context
.
Context
,
binlogMeta
*
binlogMeta
,
inFlushCh
chan
*
flushMsg
,
replica
Replica
)
*
ddNode
{
inFlushCh
<-
chan
*
flushMsg
,
replica
Replica
)
*
ddNode
{
maxQueueLength
:=
Params
.
FlowGraphMaxQueueLength
maxQueueLength
:=
Params
.
FlowGraphMaxQueueLength
maxParallelism
:=
Params
.
FlowGraphMaxParallelism
maxParallelism
:=
Params
.
FlowGraphMaxParallelism
...
...
internal/datanode/flow_graph_insert_buffer_node.go
浏览文件 @
91f5c480
...
@@ -137,6 +137,7 @@ func (ibNode *insertBufferNode) Operate(in []flowgraph.Msg) []flowgraph.Msg {
...
@@ -137,6 +137,7 @@ func (ibNode *insertBufferNode) Operate(in []flowgraph.Msg) []flowgraph.Msg {
for
_
,
pos
:=
range
iMsg
.
startPositions
{
for
_
,
pos
:=
range
iMsg
.
startPositions
{
if
pos
.
ChannelName
==
segment
.
channelName
{
if
pos
.
ChannelName
==
segment
.
channelName
{
startPosition
=
pos
startPosition
=
pos
break
}
}
}
}
if
startPosition
==
nil
{
if
startPosition
==
nil
{
...
@@ -457,6 +458,7 @@ func (ibNode *insertBufferNode) Operate(in []flowgraph.Msg) []flowgraph.Msg {
...
@@ -457,6 +458,7 @@ func (ibNode *insertBufferNode) Operate(in []flowgraph.Msg) []flowgraph.Msg {
if
ibNode
.
insertBuffer
.
full
(
currentSegID
)
{
if
ibNode
.
insertBuffer
.
full
(
currentSegID
)
{
log
.
Debug
(
". Insert Buffer full, auto flushing "
,
log
.
Debug
(
". Insert Buffer full, auto flushing "
,
zap
.
Int32
(
"num of rows"
,
ibNode
.
insertBuffer
.
size
(
currentSegID
)))
zap
.
Int32
(
"num of rows"
,
ibNode
.
insertBuffer
.
size
(
currentSegID
)))
collSch
,
err
:=
ibNode
.
getCollectionSchemaByID
(
collection
.
GetID
())
collSch
,
err
:=
ibNode
.
getCollectionSchemaByID
(
collection
.
GetID
())
if
err
!=
nil
{
if
err
!=
nil
{
log
.
Error
(
"Auto flush failed .. cannot get collection schema .."
,
zap
.
Error
(
err
))
log
.
Error
(
"Auto flush failed .. cannot get collection schema .."
,
zap
.
Error
(
err
))
...
@@ -516,16 +518,23 @@ func (ibNode *insertBufferNode) Operate(in []flowgraph.Msg) []flowgraph.Msg {
...
@@ -516,16 +518,23 @@ func (ibNode *insertBufferNode) Operate(in []flowgraph.Msg) []flowgraph.Msg {
log
.
Debug
(
".. Buffer not empty, flushing .."
)
log
.
Debug
(
".. Buffer not empty, flushing .."
)
ibNode
.
flushMap
.
Store
(
currentSegID
,
ibNode
.
insertBuffer
.
insertData
[
currentSegID
])
ibNode
.
flushMap
.
Store
(
currentSegID
,
ibNode
.
insertBuffer
.
insertData
[
currentSegID
])
delete
(
ibNode
.
insertBuffer
.
insertData
,
currentSegID
)
delete
(
ibNode
.
insertBuffer
.
insertData
,
currentSegID
)
clearFn
:=
func
()
{
finishCh
<-
false
log
.
Debug
(
".. Clearing flush Buffer .."
)
ibNode
.
flushMap
.
Delete
(
currentSegID
)
}
seg
,
err
:=
ibNode
.
replica
.
getSegmentByID
(
currentSegID
)
seg
,
err
:=
ibNode
.
replica
.
getSegmentByID
(
currentSegID
)
if
err
!=
nil
{
if
err
!=
nil
{
log
.
Error
(
"Flush failed .. cannot get segment .."
,
zap
.
Error
(
err
))
log
.
Error
(
"Flush failed .. cannot get segment .."
,
zap
.
Error
(
err
))
clearFn
()
continue
continue
}
}
collSch
,
err
:=
ibNode
.
getCollectionSchemaByID
(
seg
.
collectionID
)
collSch
,
err
:=
ibNode
.
getCollectionSchemaByID
(
seg
.
collectionID
)
if
err
!=
nil
{
if
err
!=
nil
{
log
.
Error
(
"Flush failed .. cannot get collection schema .."
,
zap
.
Error
(
err
))
log
.
Error
(
"Flush failed .. cannot get collection schema .."
,
zap
.
Error
(
err
))
clearFn
()
continue
continue
}
}
...
@@ -557,10 +566,11 @@ func (ibNode *insertBufferNode) Operate(in []flowgraph.Msg) []flowgraph.Msg {
...
@@ -557,10 +566,11 @@ func (ibNode *insertBufferNode) Operate(in []flowgraph.Msg) []flowgraph.Msg {
func
flushSegmentTxn
(
collMeta
*
etcdpb
.
CollectionMeta
,
segID
UniqueID
,
partitionID
UniqueID
,
collID
UniqueID
,
func
flushSegmentTxn
(
collMeta
*
etcdpb
.
CollectionMeta
,
segID
UniqueID
,
partitionID
UniqueID
,
collID
UniqueID
,
insertData
*
sync
.
Map
,
meta
*
binlogMeta
,
kv
kv
.
Base
,
finishCh
chan
<-
bool
)
{
insertData
*
sync
.
Map
,
meta
*
binlogMeta
,
kv
kv
.
Base
,
finishCh
chan
<-
bool
)
{
defer
func
()
{
clearFn
:=
func
(
isSuccess
bool
)
{
finishCh
<-
isSuccess
log
.
Debug
(
".. Clearing flush Buffer .."
)
log
.
Debug
(
".. Clearing flush Buffer .."
)
insertData
.
Delete
(
segID
)
insertData
.
Delete
(
segID
)
}
()
}
inCodec
:=
storage
.
NewInsertCodec
(
collMeta
)
inCodec
:=
storage
.
NewInsertCodec
(
collMeta
)
...
@@ -568,14 +578,14 @@ func flushSegmentTxn(collMeta *etcdpb.CollectionMeta, segID UniqueID, partitionI
...
@@ -568,14 +578,14 @@ func flushSegmentTxn(collMeta *etcdpb.CollectionMeta, segID UniqueID, partitionI
data
,
ok
:=
insertData
.
Load
(
segID
)
data
,
ok
:=
insertData
.
Load
(
segID
)
if
!
ok
{
if
!
ok
{
log
.
Error
(
"Flush failed ... cannot load insertData .."
)
log
.
Error
(
"Flush failed ... cannot load insertData .."
)
finishCh
<-
false
clearFn
(
false
)
return
return
}
}
binLogs
,
err
:=
inCodec
.
Serialize
(
partitionID
,
segID
,
data
.
(
*
InsertData
))
binLogs
,
err
:=
inCodec
.
Serialize
(
partitionID
,
segID
,
data
.
(
*
InsertData
))
if
err
!=
nil
{
if
err
!=
nil
{
log
.
Error
(
"Flush failed ... cannot generate binlog .."
,
zap
.
Error
(
err
))
log
.
Error
(
"Flush failed ... cannot generate binlog .."
,
zap
.
Error
(
err
))
finishCh
<-
false
clearFn
(
false
)
return
return
}
}
...
@@ -587,14 +597,14 @@ func flushSegmentTxn(collMeta *etcdpb.CollectionMeta, segID UniqueID, partitionI
...
@@ -587,14 +597,14 @@ func flushSegmentTxn(collMeta *etcdpb.CollectionMeta, segID UniqueID, partitionI
fieldID
,
err
:=
strconv
.
ParseInt
(
blob
.
GetKey
(),
10
,
64
)
fieldID
,
err
:=
strconv
.
ParseInt
(
blob
.
GetKey
(),
10
,
64
)
if
err
!=
nil
{
if
err
!=
nil
{
log
.
Error
(
"Flush failed ... cannot parse string to fieldID .."
,
zap
.
Error
(
err
))
log
.
Error
(
"Flush failed ... cannot parse string to fieldID .."
,
zap
.
Error
(
err
))
finishCh
<-
false
clearFn
(
false
)
return
return
}
}
k
,
err
:=
meta
.
genKey
(
true
,
collID
,
partitionID
,
segID
,
fieldID
)
k
,
err
:=
meta
.
genKey
(
true
,
collID
,
partitionID
,
segID
,
fieldID
)
if
err
!=
nil
{
if
err
!=
nil
{
log
.
Error
(
"Flush failed ... cannot alloc ID .."
,
zap
.
Error
(
err
))
log
.
Error
(
"Flush failed ... cannot alloc ID .."
,
zap
.
Error
(
err
))
finishCh
<-
false
clearFn
(
false
)
return
return
}
}
...
@@ -608,7 +618,7 @@ func flushSegmentTxn(collMeta *etcdpb.CollectionMeta, segID UniqueID, partitionI
...
@@ -608,7 +618,7 @@ func flushSegmentTxn(collMeta *etcdpb.CollectionMeta, segID UniqueID, partitionI
if
err
!=
nil
{
if
err
!=
nil
{
log
.
Error
(
"Flush failed ... cannot save to MinIO .."
,
zap
.
Error
(
err
))
log
.
Error
(
"Flush failed ... cannot save to MinIO .."
,
zap
.
Error
(
err
))
_
=
kv
.
MultiRemove
(
paths
)
_
=
kv
.
MultiRemove
(
paths
)
finishCh
<-
false
clearFn
(
false
)
return
return
}
}
...
@@ -617,12 +627,11 @@ func flushSegmentTxn(collMeta *etcdpb.CollectionMeta, segID UniqueID, partitionI
...
@@ -617,12 +627,11 @@ func flushSegmentTxn(collMeta *etcdpb.CollectionMeta, segID UniqueID, partitionI
if
err
!=
nil
{
if
err
!=
nil
{
log
.
Error
(
"Flush failed ... cannot save binlog paths .."
,
zap
.
Error
(
err
))
log
.
Error
(
"Flush failed ... cannot save binlog paths .."
,
zap
.
Error
(
err
))
_
=
kv
.
MultiRemove
(
paths
)
_
=
kv
.
MultiRemove
(
paths
)
finishCh
<-
false
clearFn
(
false
)
return
return
}
}
finishCh
<-
true
clearFn
(
true
)
}
}
func
(
ibNode
*
insertBufferNode
)
completeFlush
(
segID
UniqueID
,
finishCh
<-
chan
bool
)
{
func
(
ibNode
*
insertBufferNode
)
completeFlush
(
segID
UniqueID
,
finishCh
<-
chan
bool
)
{
...
...
internal/datanode/flow_graph_insert_buffer_node_test.go
浏览文件 @
91f5c480
...
@@ -40,6 +40,8 @@ func TestFlowGraphInsertBufferNode_Operate(t *testing.T) {
...
@@ -40,6 +40,8 @@ func TestFlowGraphInsertBufferNode_Operate(t *testing.T) {
replica
:=
newReplica
()
replica
:=
newReplica
()
err
=
replica
.
addCollection
(
collMeta
.
ID
,
collMeta
.
Schema
)
err
=
replica
.
addCollection
(
collMeta
.
ID
,
collMeta
.
Schema
)
require
.
NoError
(
t
,
err
)
require
.
NoError
(
t
,
err
)
err
=
replica
.
addSegment
(
1
,
collMeta
.
ID
,
0
,
Params
.
InsertChannelNames
[
0
])
require
.
NoError
(
t
,
err
)
msFactory
:=
pulsarms
.
NewFactory
()
msFactory
:=
pulsarms
.
NewFactory
()
m
:=
map
[
string
]
interface
{}{
m
:=
map
[
string
]
interface
{}{
...
@@ -64,7 +66,7 @@ func genInsertMsg() insertMsg {
...
@@ -64,7 +66,7 @@ func genInsertMsg() insertMsg {
startPos
:=
[]
*
internalpb
.
MsgPosition
{
startPos
:=
[]
*
internalpb
.
MsgPosition
{
{
{
ChannelName
:
"aaa"
,
ChannelName
:
Params
.
InsertChannelNames
[
0
]
,
MsgID
:
make
([]
byte
,
0
),
MsgID
:
make
([]
byte
,
0
),
Timestamp
:
0
,
Timestamp
:
0
,
},
},
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录