Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
7e8b203f
M
milvus
项目概览
milvus
/
milvus
12 个月 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
7e8b203f
编写于
12月 26, 2020
作者:
N
neza2017
提交者:
yefu.chen
12月 26, 2020
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Add time tick in wite node
Signed-off-by:
N
neza2017
<
yefu.chen@zilliz.com
>
上级
e316533a
变更
3
隐藏空白更改
内联
并排
Showing
3 changed file
with
46 addition
and
16 deletion
+46
-16
internal/writenode/flow_graph_insert_buffer_node.go
internal/writenode/flow_graph_insert_buffer_node.go
+44
-14
internal/writenode/param_table.go
internal/writenode/param_table.go
+1
-1
internal/writenode/param_table_test.go
internal/writenode/param_table_test.go
+1
-1
未找到文件。
internal/writenode/flow_graph_insert_buffer_node.go
浏览文件 @
7e8b203f
...
...
@@ -16,7 +16,9 @@ import (
"github.com/zilliztech/milvus-distributed/internal/kv"
etcdkv
"github.com/zilliztech/milvus-distributed/internal/kv/etcd"
miniokv
"github.com/zilliztech/milvus-distributed/internal/kv/minio"
"github.com/zilliztech/milvus-distributed/internal/msgstream"
"github.com/zilliztech/milvus-distributed/internal/proto/etcdpb"
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
"github.com/zilliztech/milvus-distributed/internal/proto/schemapb"
"github.com/zilliztech/milvus-distributed/internal/storage"
"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
...
...
@@ -34,12 +36,13 @@ type (
insertBufferNode
struct
{
BaseNode
kvClient
*
etcdkv
.
EtcdKV
insertBuffer
*
insertBuffer
minIOKV
kv
.
Base
minioPrifex
string
idAllocator
*
allocator
.
IDAllocator
outCh
chan
*
insertFlushSyncMsg
kvClient
*
etcdkv
.
EtcdKV
insertBuffer
*
insertBuffer
minIOKV
kv
.
Base
minioPrifex
string
idAllocator
*
allocator
.
IDAllocator
outCh
chan
*
insertFlushSyncMsg
pulsarWriteNodeTimeTickStream
*
msgstream
.
PulsarMsgStream
}
insertBuffer
struct
{
...
...
@@ -426,11 +429,33 @@ func (ibNode *insertBufferNode) Operate(in []*Msg) []*Msg {
// Return
}
if
err
:=
ibNode
.
writeHardTimeTick
(
iMsg
.
timeRange
.
timestampMax
);
err
!=
nil
{
log
.
Printf
(
"Error: send hard time tick into pulsar channel failed, %s
\n
"
,
err
.
Error
())
}
return
nil
}
func
newInsertBufferNode
(
ctx
context
.
Context
,
outCh
chan
*
insertFlushSyncMsg
)
*
insertBufferNode
{
func
(
ibNode
*
insertBufferNode
)
writeHardTimeTick
(
ts
Timestamp
)
error
{
msgPack
:=
msgstream
.
MsgPack
{}
timeTickMsg
:=
msgstream
.
TimeTickMsg
{
BaseMsg
:
msgstream
.
BaseMsg
{
BeginTimestamp
:
ts
,
EndTimestamp
:
ts
,
HashValues
:
[]
uint32
{
0
},
},
TimeTickMsg
:
internalpb
.
TimeTickMsg
{
MsgType
:
internalpb
.
MsgType_kTimeTick
,
PeerID
:
Params
.
WriteNodeID
,
Timestamp
:
ts
,
},
}
msgPack
.
Msgs
=
append
(
msgPack
.
Msgs
,
&
timeTickMsg
)
return
ibNode
.
pulsarWriteNodeTimeTickStream
.
Produce
(
&
msgPack
)
}
func
newInsertBufferNode
(
ctx
context
.
Context
,
outCh
chan
*
insertFlushSyncMsg
)
*
insertBufferNode
{
maxQueueLength
:=
Params
.
FlowGraphMaxQueueLength
maxParallelism
:=
Params
.
FlowGraphMaxParallelism
...
...
@@ -482,13 +507,18 @@ func newInsertBufferNode(ctx context.Context, outCh chan *insertFlushSyncMsg) *i
panic
(
err
)
}
wTt
:=
msgstream
.
NewPulsarMsgStream
(
ctx
,
1024
)
//input stream, write node time tick
wTt
.
SetPulsarClient
(
Params
.
PulsarAddress
)
wTt
.
CreatePulsarProducers
([]
string
{
Params
.
WriteNodeTimeTickChannelName
})
return
&
insertBufferNode
{
BaseNode
:
baseNode
,
kvClient
:
kvClient
,
insertBuffer
:
iBuffer
,
minIOKV
:
minIOKV
,
minioPrifex
:
minioPrefix
,
idAllocator
:
idAllocator
,
outCh
:
outCh
,
BaseNode
:
baseNode
,
kvClient
:
kvClient
,
insertBuffer
:
iBuffer
,
minIOKV
:
minIOKV
,
minioPrifex
:
minioPrefix
,
idAllocator
:
idAllocator
,
outCh
:
outCh
,
pulsarWriteNodeTimeTickStream
:
wTt
,
}
}
internal/writenode/param_table.go
浏览文件 @
7e8b203f
...
...
@@ -260,7 +260,7 @@ func (p *ParamTable) initWriteNodeTimeTickChannelName() {
if
err
!=
nil
{
panic
(
err
)
}
p
.
WriteNodeTimeTickChannelName
=
channels
p
.
WriteNodeTimeTickChannelName
=
channels
+
"-"
+
strconv
.
FormatInt
(
p
.
WriteNodeID
,
10
)
}
func
(
p
*
ParamTable
)
initSliceIndex
()
{
...
...
internal/writenode/param_table_test.go
浏览文件 @
7e8b203f
...
...
@@ -64,7 +64,7 @@ func TestParamTable_WriteNode(t *testing.T) {
t
.
Run
(
"Test timeTickChannelName"
,
func
(
t
*
testing
.
T
)
{
name
:=
Params
.
WriteNodeTimeTickChannelName
assert
.
Equal
(
t
,
name
,
"writeNodeTimeTick"
)
assert
.
Equal
(
t
,
name
,
"writeNodeTimeTick
-3
"
)
})
t
.
Run
(
"Test minioAccessKeyID"
,
func
(
t
*
testing
.
T
)
{
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录