Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
00b3fcb9
M
milvus
项目概览
milvus
/
milvus
11 个月 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
00b3fcb9
编写于
12月 29, 2021
作者:
J
Jiquan Long
提交者:
GitHub
12月 29, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Add log for flowgraph (#14441)
Signed-off-by:
N
dragondriver
<
jiquan.long@zilliz.com
>
上级
69087ff8
变更
4
显示空白变更内容
内联
并排
Showing
4 changed file
with
78 addition
and
52 deletion
+78
-52
internal/datacoord/server.go
internal/datacoord/server.go
+4
-2
internal/datacoord/util.go
internal/datacoord/util.go
+0
-50
internal/util/flowgraph/node.go
internal/util/flowgraph/node.go
+24
-0
internal/util/timerecord/time_recorder.go
internal/util/timerecord/time_recorder.go
+50
-0
未找到文件。
internal/datacoord/server.go
浏览文件 @
00b3fcb9
...
@@ -26,6 +26,8 @@ import (
...
@@ -26,6 +26,8 @@ import (
"syscall"
"syscall"
"time"
"time"
"github.com/milvus-io/milvus/internal/util/timerecord"
datanodeclient
"github.com/milvus-io/milvus/internal/distributed/datanode/client"
datanodeclient
"github.com/milvus-io/milvus/internal/distributed/datanode/client"
rootcoordclient
"github.com/milvus-io/milvus/internal/distributed/rootcoord/client"
rootcoordclient
"github.com/milvus-io/milvus/internal/distributed/rootcoord/client"
etcdkv
"github.com/milvus-io/milvus/internal/kv/etcd"
etcdkv
"github.com/milvus-io/milvus/internal/kv/etcd"
...
@@ -454,9 +456,9 @@ func (s *Server) startDataNodeTtLoop(ctx context.Context) {
...
@@ -454,9 +456,9 @@ func (s *Server) startDataNodeTtLoop(ctx context.Context) {
ttMsgStream
.
Start
()
ttMsgStream
.
Start
()
go
func
()
{
go
func
()
{
var
checker
*
LongTermChecker
var
checker
*
timerecord
.
LongTermChecker
if
enableTtChecker
{
if
enableTtChecker
{
checker
=
NewLongTermChecker
(
ctx
,
ttCheckerName
,
ttMaxInterval
,
ttCheckerWarnMsg
)
checker
=
timerecord
.
NewLongTermChecker
(
ctx
,
ttCheckerName
,
ttMaxInterval
,
ttCheckerWarnMsg
)
checker
.
Start
()
checker
.
Start
()
defer
checker
.
Stop
()
defer
checker
.
Stop
()
}
}
...
...
internal/datacoord/util.go
浏览文件 @
00b3fcb9
...
@@ -19,10 +19,8 @@ package datacoord
...
@@ -19,10 +19,8 @@ package datacoord
import
(
import
(
"context"
"context"
"errors"
"errors"
"fmt"
"time"
"time"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/proto/commonpb"
"github.com/milvus-io/milvus/internal/proto/commonpb"
"github.com/milvus-io/milvus/internal/util/tsoutil"
"github.com/milvus-io/milvus/internal/util/tsoutil"
)
)
...
@@ -68,54 +66,6 @@ func FailResponse(status *commonpb.Status, reason string) {
...
@@ -68,54 +66,6 @@ func FailResponse(status *commonpb.Status, reason string) {
status
.
Reason
=
reason
status
.
Reason
=
reason
}
}
// LongTermChecker checks we receive at least one msg in d duration. If not, checker
// will print a warn message.
type
LongTermChecker
struct
{
d
time
.
Duration
t
*
time
.
Ticker
ch
chan
struct
{}
warn
string
name
string
}
// NewLongTermChecker creates a long term checker specified name, checking interval and warning string to print
func
NewLongTermChecker
(
ctx
context
.
Context
,
name
string
,
d
time
.
Duration
,
warn
string
)
*
LongTermChecker
{
c
:=
&
LongTermChecker
{
name
:
name
,
d
:
d
,
warn
:
warn
,
ch
:
make
(
chan
struct
{}),
}
return
c
}
// Start starts the check process
func
(
c
*
LongTermChecker
)
Start
()
{
c
.
t
=
time
.
NewTicker
(
c
.
d
)
go
func
()
{
for
{
select
{
case
<-
c
.
ch
:
log
.
Warn
(
fmt
.
Sprintf
(
"long term checker [%s] shutdown"
,
c
.
name
))
return
case
<-
c
.
t
.
C
:
log
.
Warn
(
c
.
warn
)
}
}
}()
}
// Check resets the time ticker
func
(
c
*
LongTermChecker
)
Check
()
{
c
.
t
.
Reset
(
c
.
d
)
}
// Stop stops the checker
func
(
c
*
LongTermChecker
)
Stop
()
{
c
.
t
.
Stop
()
close
(
c
.
ch
)
}
func
getTimetravelReverseTime
(
ctx
context
.
Context
,
allocator
allocator
)
(
*
timetravel
,
error
)
{
func
getTimetravelReverseTime
(
ctx
context
.
Context
,
allocator
allocator
)
(
*
timetravel
,
error
)
{
ts
,
err
:=
allocator
.
allocTimestamp
(
ctx
)
ts
,
err
:=
allocator
.
allocTimestamp
(
ctx
)
if
err
!=
nil
{
if
err
!=
nil
{
...
...
internal/util/flowgraph/node.go
浏览文件 @
00b3fcb9
...
@@ -12,14 +12,23 @@
...
@@ -12,14 +12,23 @@
package
flowgraph
package
flowgraph
import
(
import
(
"context"
"fmt"
"fmt"
"sync"
"sync"
"time"
"time"
"github.com/milvus-io/milvus/internal/util/timerecord"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/log"
"go.uber.org/zap"
"go.uber.org/zap"
)
)
const
(
// TODO: better to be configured
nodeCtxTtInterval
=
2
*
time
.
Minute
enableTtChecker
=
true
)
// Node is the interface defines the behavior of flowgraph
// Node is the interface defines the behavior of flowgraph
type
Node
interface
{
type
Node
interface
{
Name
()
string
Name
()
string
...
@@ -61,6 +70,17 @@ func (nodeCtx *nodeCtx) Start(wg *sync.WaitGroup) {
...
@@ -61,6 +70,17 @@ func (nodeCtx *nodeCtx) Start(wg *sync.WaitGroup) {
// 2. invoke node.Operate
// 2. invoke node.Operate
// 3. deliver the Operate result to downstream nodes
// 3. deliver the Operate result to downstream nodes
func
(
nodeCtx
*
nodeCtx
)
work
()
{
func
(
nodeCtx
*
nodeCtx
)
work
()
{
// TODO: necessary to check every node?
name
:=
fmt
.
Sprintf
(
"nodeCtxTtChecker-%s"
,
nodeCtx
.
node
.
Name
())
warn
:=
fmt
.
Sprintf
(
"node %s haven't received input for %f minutes"
,
nodeCtx
.
node
.
Name
(),
nodeCtxTtInterval
.
Minutes
())
var
checker
*
timerecord
.
LongTermChecker
if
enableTtChecker
{
checker
=
timerecord
.
NewLongTermChecker
(
context
.
Background
(),
name
,
nodeCtxTtInterval
,
warn
)
checker
.
Start
()
defer
checker
.
Stop
()
}
for
{
for
{
select
{
select
{
case
<-
nodeCtx
.
closeCh
:
case
<-
nodeCtx
.
closeCh
:
...
@@ -76,6 +96,10 @@ func (nodeCtx *nodeCtx) work() {
...
@@ -76,6 +96,10 @@ func (nodeCtx *nodeCtx) work() {
n
:=
nodeCtx
.
node
n
:=
nodeCtx
.
node
res
=
n
.
Operate
(
inputs
)
res
=
n
.
Operate
(
inputs
)
if
enableTtChecker
{
checker
.
Check
()
}
downstreamLength
:=
len
(
nodeCtx
.
downstreamInputChanIdx
)
downstreamLength
:=
len
(
nodeCtx
.
downstreamInputChanIdx
)
if
len
(
nodeCtx
.
downstream
)
<
downstreamLength
{
if
len
(
nodeCtx
.
downstream
)
<
downstreamLength
{
log
.
Warn
(
""
,
zap
.
Any
(
"nodeCtx.downstream length"
,
len
(
nodeCtx
.
downstream
)))
log
.
Warn
(
""
,
zap
.
Any
(
"nodeCtx.downstream length"
,
len
(
nodeCtx
.
downstream
)))
...
...
internal/util/timerecord/time_recorder.go
浏览文件 @
00b3fcb9
...
@@ -12,6 +12,8 @@
...
@@ -12,6 +12,8 @@
package
timerecord
package
timerecord
import
(
import
(
"context"
"fmt"
"strconv"
"strconv"
"time"
"time"
...
@@ -75,3 +77,51 @@ func (tr *TimeRecorder) printTimeRecord(msg string, span time.Duration) {
...
@@ -75,3 +77,51 @@ func (tr *TimeRecorder) printTimeRecord(msg string, span time.Duration) {
str
+=
"ms)"
str
+=
"ms)"
log
.
Debug
(
str
)
log
.
Debug
(
str
)
}
}
// LongTermChecker checks we receive at least one msg in d duration. If not, checker
// will print a warn message.
type
LongTermChecker
struct
{
d
time
.
Duration
t
*
time
.
Ticker
ch
chan
struct
{}
warn
string
name
string
}
// NewLongTermChecker creates a long term checker specified name, checking interval and warning string to print
func
NewLongTermChecker
(
ctx
context
.
Context
,
name
string
,
d
time
.
Duration
,
warn
string
)
*
LongTermChecker
{
c
:=
&
LongTermChecker
{
name
:
name
,
d
:
d
,
warn
:
warn
,
ch
:
make
(
chan
struct
{}),
}
return
c
}
// Start starts the check process
func
(
c
*
LongTermChecker
)
Start
()
{
c
.
t
=
time
.
NewTicker
(
c
.
d
)
go
func
()
{
for
{
select
{
case
<-
c
.
ch
:
log
.
Warn
(
fmt
.
Sprintf
(
"long term checker [%s] shutdown"
,
c
.
name
))
return
case
<-
c
.
t
.
C
:
log
.
Warn
(
c
.
warn
)
}
}
}()
}
// Check resets the time ticker
func
(
c
*
LongTermChecker
)
Check
()
{
c
.
t
.
Reset
(
c
.
d
)
}
// Stop stops the checker
func
(
c
*
LongTermChecker
)
Stop
()
{
c
.
t
.
Stop
()
close
(
c
.
ch
)
}
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录