Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
cbbeb2a3
M
milvus
项目概览
milvus
/
milvus
11 个月 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
cbbeb2a3
编写于
11月 27, 2021
作者:
C
Cai Yudong
提交者:
GitHub
11月 27, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Add debug log when proxy idle for a long time (#12319)
Signed-off-by:
N
yudong.cai
<
yudong.cai@zilliz.com
>
上级
48bdeb56
变更
3
隐藏空白更改
内联
并排
Showing
3 changed file
with
64 addition
and
53 deletion
+64
-53
internal/rootcoord/root_coord_test.go
internal/rootcoord/root_coord_test.go
+5
-8
internal/rootcoord/timeticksync.go
internal/rootcoord/timeticksync.go
+57
-43
internal/rootcoord/timeticksync_test.go
internal/rootcoord/timeticksync_test.go
+2
-2
未找到文件。
internal/rootcoord/root_coord_test.go
浏览文件 @
cbbeb2a3
...
...
@@ -720,14 +720,11 @@ func TestRootCoord(t *testing.T) {
assert
.
Equal
(
t
,
len
(
core
.
chanTimeTick
.
proxyTimeTick
),
2
)
pt
,
ok
:=
core
.
chanTimeTick
.
proxyTimeTick
[
core
.
session
.
ServerID
]
assert
.
True
(
t
,
ok
)
assert
.
Equal
(
t
,
shardsNum
,
int32
(
len
(
pt
.
in
.
ChannelNames
)))
assert
.
Equal
(
t
,
shardsNum
,
int32
(
len
(
pt
.
in
.
Timestamps
)))
assert
.
Equal
(
t
,
shardsNum
,
int32
(
len
(
pt
.
timeTick
)))
assert
.
ElementsMatch
(
t
,
pt
.
in
.
ChannelNames
,
createMeta
.
PhysicalChannelNames
)
assert
.
Equal
(
t
,
pt
.
in
.
Timestamps
[
0
],
pt
.
in
.
Timestamps
[
1
])
assert
.
Equal
(
t
,
pt
.
in
.
Timestamps
[
0
],
pt
.
in
.
DefaultTimestamp
)
assert
.
Equal
(
t
,
pt
.
timeTick
[
pt
.
in
.
ChannelNames
[
0
]],
pt
.
in
.
DefaultTimestamp
)
assert
.
Equal
(
t
,
pt
.
timeTick
[
pt
.
in
.
ChannelNames
[
1
]],
pt
.
in
.
DefaultTimestamp
)
assert
.
Equal
(
t
,
shardsNum
,
int32
(
len
(
pt
.
chanTs
)))
for
chanName
,
ts
:=
range
pt
.
chanTs
{
assert
.
Contains
(
t
,
createMeta
.
PhysicalChannelNames
,
chanName
)
assert
.
Equal
(
t
,
pt
.
defaultTs
,
ts
)
}
core
.
chanTimeTick
.
lock
.
Unlock
()
// check DD operation info
...
...
internal/rootcoord/timeticksync.go
浏览文件 @
cbbeb2a3
...
...
@@ -37,8 +37,8 @@ type timetickSync struct {
deltaChannels
*
dmlChannels
// used for delete
lock
sync
.
Mutex
proxyTimeTick
map
[
typeutil
.
UniqueID
]
*
chan
nelTimeTick
Msg
sendChan
chan
map
[
typeutil
.
UniqueID
]
*
chan
nelTimeTick
Msg
proxyTimeTick
map
[
typeutil
.
UniqueID
]
*
chan
Ts
Msg
sendChan
chan
map
[
typeutil
.
UniqueID
]
*
chan
Ts
Msg
// record ddl timetick info
ddlLock
sync
.
RWMutex
...
...
@@ -46,28 +46,29 @@ type timetickSync struct {
ddlTsSet
map
[
typeutil
.
Timestamp
]
struct
{}
}
type
channelTimeTickMsg
struct
{
in
*
internalpb
.
ChannelTimeTickMsg
timeTick
map
[
string
]
typeutil
.
Timestamp
type
chanTsMsg
struct
{
chanTs
map
[
string
]
typeutil
.
Timestamp
defaultTs
typeutil
.
Timestamp
cnt
int64
}
func
newChannelTimeTickMsg
(
in
*
internalpb
.
ChannelTimeTickMsg
)
*
channelTimeTickMsg
{
msg
:=
&
channelTimeTickMsg
{
in
:
in
,
timeTick
:
make
(
map
[
string
]
typeutil
.
Timestamp
),
func
newChanTsMsg
(
in
*
internalpb
.
ChannelTimeTickMsg
,
cnt
int64
)
*
chanTsMsg
{
msg
:=
&
chanTsMsg
{
chanTs
:
make
(
map
[
string
]
typeutil
.
Timestamp
),
defaultTs
:
in
.
DefaultTimestamp
,
cnt
:
cnt
,
}
for
idx
:=
range
in
.
ChannelNames
{
msg
.
timeTick
[
in
.
ChannelNames
[
idx
]]
=
in
.
Timestamps
[
idx
]
msg
.
chanTs
[
in
.
ChannelNames
[
idx
]]
=
in
.
Timestamps
[
idx
]
}
return
msg
}
func
(
c
*
channelTimeTickMsg
)
getTimetick
(
channelName
string
)
typeutil
.
Timestamp
{
tt
,
ok
:=
c
.
timeTick
[
channelName
]
if
ok
{
return
tt
func
(
c
*
chanTsMsg
)
getTimetick
(
channelName
string
)
typeutil
.
Timestamp
{
if
ts
,
ok
:=
c
.
chanTs
[
channelName
];
ok
{
return
ts
}
return
c
.
in
.
DefaultTimestamp
return
c
.
defaultTs
}
func
newTimeTickSync
(
ctx
context
.
Context
,
session
*
sessionutil
.
Session
,
factory
msgstream
.
Factory
,
chanMap
map
[
typeutil
.
UniqueID
][]
string
)
*
timetickSync
{
...
...
@@ -102,8 +103,8 @@ func newTimeTickSync(ctx context.Context, session *sessionutil.Session, factory
deltaChannels
:
deltaChannels
,
lock
:
sync
.
Mutex
{},
proxyTimeTick
:
make
(
map
[
typeutil
.
UniqueID
]
*
chan
nelTimeTick
Msg
),
sendChan
:
make
(
chan
map
[
typeutil
.
UniqueID
]
*
chan
nelTimeTick
Msg
,
16
),
proxyTimeTick
:
make
(
map
[
typeutil
.
UniqueID
]
*
chan
Ts
Msg
),
sendChan
:
make
(
chan
map
[
typeutil
.
UniqueID
]
*
chan
Ts
Msg
,
16
),
ddlLock
:
sync
.
RWMutex
{},
ddlMinTs
:
typeutil
.
Timestamp
(
math
.
MaxUint64
),
...
...
@@ -117,13 +118,31 @@ func (t *timetickSync) sendToChannel() {
if
len
(
t
.
proxyTimeTick
)
==
0
{
return
}
for
_
,
v
:=
range
t
.
proxyTimeTick
{
// detect whether rootcoord receives ttMsg from all proxy nodes
maxCnt
:=
int64
(
0
)
idleProxyList
:=
make
([]
typeutil
.
UniqueID
,
0
)
for
id
,
v
:=
range
t
.
proxyTimeTick
{
if
v
==
nil
{
return
idleProxyList
=
append
(
idleProxyList
,
id
)
}
else
{
if
maxCnt
<
v
.
cnt
{
maxCnt
=
v
.
cnt
}
}
}
if
len
(
idleProxyList
)
>
0
{
// give warning every 2 second if not get ttMsg from proxy nodes
if
maxCnt
%
10
==
0
{
log
.
Warn
(
"proxy idle for long time"
,
zap
.
Any
(
"proxy list"
,
idleProxyList
),
zap
.
Int64
(
"idle time"
,
int64
(
Params
.
TimeTickInterval
)
*
maxCnt
))
}
return
}
// clear proxyTimeTick and send a clone
ptt
:=
make
(
map
[
typeutil
.
UniqueID
]
*
chan
nelTimeTick
Msg
)
ptt
:=
make
(
map
[
typeutil
.
UniqueID
]
*
chan
Ts
Msg
)
for
k
,
v
:=
range
t
.
proxyTimeTick
{
ptt
[
k
]
=
v
t
.
proxyTimeTick
[
k
]
=
nil
...
...
@@ -204,24 +223,20 @@ func (t *timetickSync) updateTimeTick(in *internalpb.ChannelTimeTickMsg, reason
}
if
in
.
Base
.
SourceID
==
t
.
session
.
ServerID
{
if
prev
!=
nil
&&
in
.
DefaultTimestamp
<=
prev
.
in
.
DefaultTimestamp
{
if
prev
!=
nil
&&
in
.
DefaultTimestamp
<=
prev
.
defaultTs
{
log
.
Debug
(
"timestamp go back"
,
zap
.
Int64
(
"source id"
,
in
.
Base
.
SourceID
),
zap
.
Uint64
(
"curr ts"
,
in
.
DefaultTimestamp
),
zap
.
Uint64
(
"prev ts"
,
prev
.
in
.
DefaultTimestamp
),
zap
.
Uint64
(
"prev ts"
,
prev
.
defaultTs
),
zap
.
String
(
"reason"
,
reason
))
return
nil
}
}
if
in
.
DefaultTimestamp
==
0
{
mints
:=
minTimeTick
(
in
.
Timestamps
...
)
log
.
Debug
(
"default time stamp is zero, set it to the min value of inputs"
,
zap
.
Int64
(
"proxy id"
,
in
.
Base
.
SourceID
),
zap
.
Uint64
(
"min ts"
,
mints
))
in
.
DefaultTimestamp
=
mints
}
t
.
proxyTimeTick
[
in
.
Base
.
SourceID
]
=
newChannelTimeTickMsg
(
in
)
//log.Debug("update proxyTimeTick", zap.Int64("source id", in.Base.SourceID),
// zap.Any("Ts", in.Timestamps), zap.Uint64("inTs", in.DefaultTimestamp), zap.String("reason", reason))
if
prev
==
nil
{
t
.
proxyTimeTick
[
in
.
Base
.
SourceID
]
=
newChanTsMsg
(
in
,
1
)
}
else
{
t
.
proxyTimeTick
[
in
.
Base
.
SourceID
]
=
newChanTsMsg
(
in
,
prev
.
cnt
+
1
)
}
t
.
sendToChannel
()
return
nil
...
...
@@ -268,36 +283,35 @@ func (t *timetickSync) startWatch(wg *sync.WaitGroup) {
// reduce each channel to get min timestamp
local
:=
proxyTimetick
[
t
.
session
.
ServerID
]
if
len
(
local
.
in
.
ChannelName
s
)
==
0
{
if
len
(
local
.
chanT
s
)
==
0
{
continue
}
hdr
:=
fmt
.
Sprintf
(
"send ts to %d channels"
,
len
(
local
.
in
.
ChannelName
s
))
hdr
:=
fmt
.
Sprintf
(
"send ts to %d channels"
,
len
(
local
.
chanT
s
))
tr
:=
timerecord
.
NewTimeRecorder
(
hdr
)
wg
:=
sync
.
WaitGroup
{}
for
_
,
chanName
:=
range
local
.
in
.
ChannelName
s
{
for
chanName
,
ts
:=
range
local
.
chanT
s
{
wg
.
Add
(
1
)
go
func
(
chanName
string
)
{
mints
:=
local
.
getTimetick
(
chanName
)
go
func
(
chanName
string
,
ts
typeutil
.
Timestamp
)
{
mints
:=
ts
for
_
,
tt
:=
range
proxyTimetick
{
t
s
:=
tt
.
getTimetick
(
chanName
)
if
t
s
<
mints
{
mints
=
t
s
currT
s
:=
tt
.
getTimetick
(
chanName
)
if
currT
s
<
mints
{
mints
=
currT
s
}
}
if
err
:=
t
.
sendTimeTickToChannel
([]
string
{
chanName
},
mints
);
err
!=
nil
{
log
.
Debug
(
"SendTimeTickToChannel fail"
,
zap
.
Error
(
err
))
}
wg
.
Done
()
}(
chanName
)
}(
chanName
,
ts
)
}
wg
.
Wait
()
span
:=
tr
.
ElapseSpan
()
// rootcoord send tt msg to all channels every 200ms by default
if
span
.
Milliseconds
()
>
200
{
if
span
.
Milliseconds
()
>
int64
(
Params
.
TimeTickInterval
)
{
log
.
Warn
(
"rootcoord send tt to all channels too slowly"
,
zap
.
Int
(
"chanNum"
,
len
(
local
.
in
.
ChannelNames
)),
zap
.
Int64
(
"span"
,
span
.
Milliseconds
()))
zap
.
Int
(
"chanNum"
,
len
(
local
.
chanTs
)),
zap
.
Int64
(
"span"
,
span
.
Milliseconds
()))
}
}
}
...
...
internal/rootcoord/timeticksync_test.go
浏览文件 @
cbbeb2a3
...
...
@@ -58,7 +58,7 @@ func TestTimetickSync(t *testing.T) {
MsgType
:
commonpb
.
MsgType_TimeTick
,
},
}
ttSync
.
proxyTimeTick
[
1
]
=
newChan
nelTimeTickMsg
(
msg
)
ttSync
.
proxyTimeTick
[
1
]
=
newChan
TsMsg
(
msg
,
1
)
ttSync
.
sendToChannel
()
})
...
...
@@ -87,7 +87,7 @@ func TestTimetickSync(t *testing.T) {
msg
.
Timestamps
=
append
(
msg
.
Timestamps
,
uint64
(
2
))
msg
.
DefaultTimestamp
=
uint64
(
200
)
cttMsg
:=
newChan
nelTimeTickMsg
(
msg
)
cttMsg
:=
newChan
TsMsg
(
msg
,
1
)
ttSync
.
proxyTimeTick
[
msg
.
Base
.
SourceID
]
=
cttMsg
ttSync
.
ddlMinTs
=
uint64
(
100
)
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录