Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
810be533
M
milvus
项目概览
milvus
/
milvus
11 个月 前同步成功
通知
260
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
810be533
编写于
1月 06, 2021
作者:
B
bigsheeper
提交者:
yefu.chen
1月 06, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Add gcNode, and remove msg's downstream index
Signed-off-by:
N
bigsheeper
<
yihao.dai@zilliz.com
>
上级
1cfc2ff0
变更
10
隐藏空白更改
内联
并排
Showing
10 changed file
with
150 addition
and
57 deletion
+150
-57
Makefile
Makefile
+4
-8
internal/querynode/data_sync_service.go
internal/querynode/data_sync_service.go
+11
-1
internal/querynode/flow_graph_dd_node.go
internal/querynode/flow_graph_dd_node.go
+22
-10
internal/querynode/flow_graph_filter_dm_node.go
internal/querynode/flow_graph_filter_dm_node.go
+22
-2
internal/querynode/flow_graph_gc_node.go
internal/querynode/flow_graph_gc_node.go
+61
-0
internal/querynode/flow_graph_insert_node.go
internal/querynode/flow_graph_insert_node.go
+1
-0
internal/querynode/flow_graph_message.go
internal/querynode/flow_graph_message.go
+23
-18
internal/querynode/flow_graph_service_time_node.go
internal/querynode/flow_graph_service_time_node.go
+6
-1
internal/util/flowgraph/message.go
internal/util/flowgraph/message.go
+0
-1
internal/writenode/flow_graph_message.go
internal/writenode/flow_graph_message.go
+0
-16
未找到文件。
Makefile
浏览文件 @
810be533
...
...
@@ -127,7 +127,6 @@ install: all
@
mkdir
-p
$(GOPATH)
/bin
&&
cp
-f
$(PWD)
/bin/master
$(GOPATH)
/bin/master
@
mkdir
-p
$(GOPATH)
/bin
&&
cp
-f
$(PWD)
/bin/proxy
$(GOPATH)
/bin/proxy
@
mkdir
-p
$(GOPATH)
/bin
&&
cp
-f
$(PWD)
/bin/writenode
$(GOPATH)
/bin/writenode
@
mkdir
-p
$(GOPATH)
/bin
&&
cp
-f
$(PWD)
/bin/indexbuilder
$(GOPATH)
/bin/indexbuilder
@
mkdir
-p
$(LIBRARY_PATH)
&&
cp
-f
$(PWD)
/internal/core/output/lib/
*
$(LIBRARY_PATH)
@
echo
"Installation successful."
...
...
@@ -135,10 +134,7 @@ clean:
@
echo
"Cleaning up all the generated files"
@
find
.
-name
'*.test'
| xargs
rm
-fv
@
find
.
-name
'*~'
| xargs
rm
-fv
@
rm
-rf
bin/
@
rm
-rf
lib/
@
rm
-rf
$(GOPATH)
/bin/master
@
rm
-rf
$(GOPATH)
/bin/proxy
@
rm
-rf
$(GOPATH)
/bin/querynode
@
rm
-rf
$(GOPATH)
/bin/writenode
@
rm
-rf
$(GOPATH)
/bin/indexbuilder
@
rm
-rvf
querynode
@
rm
-rvf
master
@
rm
-rvf
proxy
@
rm
-rvf
writenode
internal/querynode/data_sync_service.go
浏览文件 @
810be533
...
...
@@ -48,6 +48,7 @@ func (dsService *dataSyncService) initNodes() {
var
insertNode
Node
=
newInsertNode
(
dsService
.
replica
)
var
serviceTimeNode
Node
=
newServiceTimeNode
(
dsService
.
replica
)
var
gcNode
Node
=
newGCNode
(
dsService
.
replica
)
dsService
.
fg
.
AddNode
(
&
dmStreamNode
)
dsService
.
fg
.
AddNode
(
&
ddStreamNode
)
...
...
@@ -57,6 +58,7 @@ func (dsService *dataSyncService) initNodes() {
dsService
.
fg
.
AddNode
(
&
insertNode
)
dsService
.
fg
.
AddNode
(
&
serviceTimeNode
)
dsService
.
fg
.
AddNode
(
&
gcNode
)
// dmStreamNode
var
err
=
dsService
.
fg
.
SetEdges
(
dmStreamNode
.
Name
(),
...
...
@@ -106,9 +108,17 @@ func (dsService *dataSyncService) initNodes() {
// serviceTimeNode
err
=
dsService
.
fg
.
SetEdges
(
serviceTimeNode
.
Name
(),
[]
string
{
insertNode
.
Name
()},
[]
string
{},
[]
string
{
gcNode
.
Name
()
},
)
if
err
!=
nil
{
log
.
Fatal
(
"set edges failed in node:"
,
serviceTimeNode
.
Name
())
}
// gcNode
err
=
dsService
.
fg
.
SetEdges
(
gcNode
.
Name
(),
[]
string
{
serviceTimeNode
.
Name
()},
[]
string
{})
if
err
!=
nil
{
log
.
Fatal
(
"set edges failed in node:"
,
gcNode
.
Name
())
}
}
internal/querynode/flow_graph_dd_node.go
浏览文件 @
810be533
...
...
@@ -44,6 +44,11 @@ func (ddNode *ddNode) Operate(in []*Msg) []*Msg {
},
}
ddNode
.
ddMsg
=
&
ddMsg
gcRecord
:=
gcRecord
{
collections
:
make
([]
UniqueID
,
0
),
partitions
:
make
([]
partitionWithID
,
0
),
}
ddNode
.
ddMsg
.
gcRecord
=
&
gcRecord
// sort tsMessages
tsMessages
:=
msMsg
.
TsMessages
()
...
...
@@ -115,11 +120,11 @@ func (ddNode *ddNode) createCollection(msg *msgstream.CreateCollectionMsg) {
func
(
ddNode
*
ddNode
)
dropCollection
(
msg
*
msgstream
.
DropCollectionMsg
)
{
collectionID
:=
msg
.
CollectionID
err
:=
ddNode
.
replica
.
removeCollection
(
collectionID
)
if
err
!=
nil
{
log
.
Println
(
err
)
return
}
//
err := ddNode.replica.removeCollection(collectionID)
//
if err != nil {
//
log.Println(err)
//
return
//
}
collectionName
:=
msg
.
CollectionName
.
CollectionName
ddNode
.
ddMsg
.
collectionRecords
[
collectionName
]
=
append
(
ddNode
.
ddMsg
.
collectionRecords
[
collectionName
],
...
...
@@ -127,6 +132,8 @@ func (ddNode *ddNode) dropCollection(msg *msgstream.DropCollectionMsg) {
createOrDrop
:
false
,
timestamp
:
msg
.
Timestamp
,
})
ddNode
.
ddMsg
.
gcRecord
.
collections
=
append
(
ddNode
.
ddMsg
.
gcRecord
.
collections
,
collectionID
)
}
func
(
ddNode
*
ddNode
)
createPartition
(
msg
*
msgstream
.
CreatePartitionMsg
)
{
...
...
@@ -150,17 +157,22 @@ func (ddNode *ddNode) dropPartition(msg *msgstream.DropPartitionMsg) {
collectionID
:=
msg
.
CollectionID
partitionTag
:=
msg
.
PartitionName
.
Tag
err
:=
ddNode
.
replica
.
removePartition
(
collectionID
,
partitionTag
)
if
err
!=
nil
{
log
.
Println
(
err
)
return
}
//
err := ddNode.replica.removePartition(collectionID, partitionTag)
//
if err != nil {
//
log.Println(err)
//
return
//
}
ddNode
.
ddMsg
.
partitionRecords
[
partitionTag
]
=
append
(
ddNode
.
ddMsg
.
partitionRecords
[
partitionTag
],
metaOperateRecord
{
createOrDrop
:
false
,
timestamp
:
msg
.
Timestamp
,
})
ddNode
.
ddMsg
.
gcRecord
.
partitions
=
append
(
ddNode
.
ddMsg
.
gcRecord
.
partitions
,
partitionWithID
{
partitionTag
:
partitionTag
,
collectionID
:
collectionID
,
})
}
func
newDDNode
(
replica
collectionReplica
)
*
ddNode
{
...
...
internal/querynode/flow_graph_filter_dm_node.go
浏览文件 @
810be533
...
...
@@ -2,6 +2,7 @@ package querynode
import
(
"log"
"math"
"github.com/zilliztech/milvus-distributed/internal/msgstream"
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
...
...
@@ -59,6 +60,7 @@ func (fdmNode *filterDmNode) Operate(in []*Msg) []*Msg {
}
}
iMsg
.
gcRecord
=
ddMsg
.
gcRecord
var
res
Msg
=
&
iMsg
return
[]
*
Msg
{
&
res
}
}
...
...
@@ -81,17 +83,35 @@ func (fdmNode *filterDmNode) filterInvalidInsertMessage(msg *msgstream.InsertMsg
log
.
Println
(
"Error, misaligned messages detected"
)
return
nil
}
tmpTimestamps
:=
make
([]
Timestamp
,
0
)
tmpRowIDs
:=
make
([]
int64
,
0
)
tmpRowData
:=
make
([]
*
commonpb
.
Blob
,
0
)
targetTimestamp
:=
records
[
len
(
records
)
-
1
]
.
timestamp
// calculate valid time range
timeBegin
:=
Timestamp
(
0
)
timeEnd
:=
Timestamp
(
math
.
MaxUint64
)
for
_
,
record
:=
range
records
{
if
record
.
createOrDrop
&&
timeBegin
<
record
.
timestamp
{
timeBegin
=
record
.
timestamp
}
if
!
record
.
createOrDrop
&&
timeEnd
>
record
.
timestamp
{
timeEnd
=
record
.
timestamp
}
}
for
i
,
t
:=
range
msg
.
Timestamps
{
if
t
>=
t
argetTimestamp
{
if
t
>=
t
imeBegin
&&
t
<=
timeEnd
{
tmpTimestamps
=
append
(
tmpTimestamps
,
t
)
tmpRowIDs
=
append
(
tmpRowIDs
,
msg
.
RowIDs
[
i
])
tmpRowData
=
append
(
tmpRowData
,
msg
.
RowData
[
i
])
}
}
if
len
(
tmpRowIDs
)
<=
0
{
return
nil
}
msg
.
Timestamps
=
tmpTimestamps
msg
.
RowIDs
=
tmpRowIDs
msg
.
RowData
=
tmpRowData
...
...
internal/querynode/flow_graph_gc_node.go
0 → 100644
浏览文件 @
810be533
package
querynode
import
(
"log"
)
type
gcNode
struct
{
BaseNode
replica
collectionReplica
}
func
(
gcNode
*
gcNode
)
Name
()
string
{
return
"gcNode"
}
func
(
gcNode
*
gcNode
)
Operate
(
in
[]
*
Msg
)
[]
*
Msg
{
//fmt.Println("Do gcNode operation")
if
len
(
in
)
!=
1
{
log
.
Println
(
"Invalid operate message input in gcNode, input length = "
,
len
(
in
))
// TODO: add error handling
}
gcMsg
,
ok
:=
(
*
in
[
0
])
.
(
*
gcMsg
)
if
!
ok
{
log
.
Println
(
"type assertion failed for gcMsg"
)
// TODO: add error handling
}
// drop collections
for
_
,
collectionID
:=
range
gcMsg
.
gcRecord
.
collections
{
err
:=
gcNode
.
replica
.
removeCollection
(
collectionID
)
if
err
!=
nil
{
log
.
Println
(
err
)
}
}
// drop partitions
for
_
,
partition
:=
range
gcMsg
.
gcRecord
.
partitions
{
err
:=
gcNode
.
replica
.
removePartition
(
partition
.
collectionID
,
partition
.
partitionTag
)
if
err
!=
nil
{
log
.
Println
(
err
)
}
}
return
nil
}
func
newGCNode
(
replica
collectionReplica
)
*
gcNode
{
maxQueueLength
:=
Params
.
FlowGraphMaxQueueLength
maxParallelism
:=
Params
.
FlowGraphMaxParallelism
baseNode
:=
BaseNode
{}
baseNode
.
SetMaxQueueLength
(
maxQueueLength
)
baseNode
.
SetMaxParallelism
(
maxParallelism
)
return
&
gcNode
{
BaseNode
:
baseNode
,
replica
:
replica
,
}
}
internal/querynode/flow_graph_insert_node.go
浏览文件 @
810be533
...
...
@@ -90,6 +90,7 @@ func (iNode *insertNode) Operate(in []*Msg) []*Msg {
wg
.
Wait
()
var
res
Msg
=
&
serviceTimeMsg
{
gcRecord
:
iMsg
.
gcRecord
,
timeRange
:
iMsg
.
timeRange
,
}
return
[]
*
Msg
{
&
res
}
...
...
internal/querynode/flow_graph_message.go
浏览文件 @
810be533
...
...
@@ -16,6 +16,7 @@ type key2SegMsg struct {
type
ddMsg
struct
{
collectionRecords
map
[
string
][]
metaOperateRecord
partitionRecords
map
[
string
][]
metaOperateRecord
gcRecord
*
gcRecord
timeRange
TimeRange
}
...
...
@@ -26,6 +27,7 @@ type metaOperateRecord struct {
type
insertMsg
struct
{
insertMessages
[]
*
msgstream
.
InsertMsg
gcRecord
*
gcRecord
timeRange
TimeRange
}
...
...
@@ -35,6 +37,12 @@ type deleteMsg struct {
}
type
serviceTimeMsg
struct
{
gcRecord
*
gcRecord
timeRange
TimeRange
}
type
gcMsg
struct
{
gcRecord
*
gcRecord
timeRange
TimeRange
}
...
...
@@ -55,42 +63,39 @@ type DeletePreprocessData struct {
count
int32
}
func
(
ksMsg
*
key2SegMsg
)
TimeTick
()
Timestamp
{
return
ksMsg
.
timeRange
.
timestampMax
// TODO: replace partitionWithID by partition id
type
partitionWithID
struct
{
partitionTag
string
collectionID
UniqueID
}
func
(
ksMsg
*
key2SegMsg
)
DownStreamNodeIdx
()
int
{
return
0
type
gcRecord
struct
{
// collections and partitions to be dropped
collections
[]
UniqueID
// TODO: use partition id
partitions
[]
partitionWithID
}
func
(
suMsg
*
dd
Msg
)
TimeTick
()
Timestamp
{
return
su
Msg
.
timeRange
.
timestampMax
func
(
ksMsg
*
key2Seg
Msg
)
TimeTick
()
Timestamp
{
return
ks
Msg
.
timeRange
.
timestampMax
}
func
(
suMsg
*
ddMsg
)
DownStreamNodeIdx
()
int
{
return
0
func
(
suMsg
*
ddMsg
)
TimeTick
()
Timestamp
{
return
suMsg
.
timeRange
.
timestampMax
}
func
(
iMsg
*
insertMsg
)
TimeTick
()
Timestamp
{
return
iMsg
.
timeRange
.
timestampMax
}
func
(
iMsg
*
insertMsg
)
DownStreamNodeIdx
()
int
{
return
0
}
func
(
dMsg
*
deleteMsg
)
TimeTick
()
Timestamp
{
return
dMsg
.
timeRange
.
timestampMax
}
func
(
dMsg
*
deleteMsg
)
DownStreamNodeIdx
()
int
{
return
0
}
func
(
stMsg
*
serviceTimeMsg
)
TimeTick
()
Timestamp
{
return
stMsg
.
timeRange
.
timestampMax
}
func
(
stMsg
*
serviceTimeMsg
)
DownStreamNodeIdx
()
int
{
return
0
func
(
gcMsg
*
gcMsg
)
TimeTick
()
Timestamp
{
return
gcMsg
.
timeRange
.
timestampMax
}
internal/querynode/flow_graph_service_time_node.go
浏览文件 @
810be533
...
...
@@ -30,7 +30,12 @@ func (stNode *serviceTimeNode) Operate(in []*Msg) []*Msg {
// update service time
stNode
.
replica
.
getTSafe
()
.
set
(
serviceTimeMsg
.
timeRange
.
timestampMax
)
//fmt.Println("update tSafe to:", getPhysicalTime(serviceTimeMsg.timeRange.timestampMax))
return
nil
var
res
Msg
=
&
gcMsg
{
gcRecord
:
serviceTimeMsg
.
gcRecord
,
timeRange
:
serviceTimeMsg
.
timeRange
,
}
return
[]
*
Msg
{
&
res
}
}
func
newServiceTimeNode
(
replica
collectionReplica
)
*
serviceTimeNode
{
...
...
internal/util/flowgraph/message.go
浏览文件 @
810be533
...
...
@@ -4,7 +4,6 @@ import "github.com/zilliztech/milvus-distributed/internal/msgstream"
type
Msg
interface
{
TimeTick
()
Timestamp
DownStreamNodeIdx
()
int
}
type
MsgStreamMsg
struct
{
...
...
internal/writenode/flow_graph_message.go
浏览文件 @
810be533
...
...
@@ -46,30 +46,14 @@ func (ksMsg *key2SegMsg) TimeTick() Timestamp {
return
ksMsg
.
timeRange
.
timestampMax
}
func
(
ksMsg
*
key2SegMsg
)
DownStreamNodeIdx
()
int
{
return
0
}
func
(
suMsg
*
ddMsg
)
TimeTick
()
Timestamp
{
return
suMsg
.
timeRange
.
timestampMax
}
func
(
suMsg
*
ddMsg
)
DownStreamNodeIdx
()
int
{
return
0
}
func
(
iMsg
*
insertMsg
)
TimeTick
()
Timestamp
{
return
iMsg
.
timeRange
.
timestampMax
}
func
(
iMsg
*
insertMsg
)
DownStreamNodeIdx
()
int
{
return
0
}
func
(
dMsg
*
deleteMsg
)
TimeTick
()
Timestamp
{
return
dMsg
.
timeRange
.
timestampMax
}
func
(
dMsg
*
deleteMsg
)
DownStreamNodeIdx
()
int
{
return
0
}
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录