Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
bfb7ca0d
M
milvus
项目概览
milvus
/
milvus
11 个月 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
bfb7ca0d
编写于
4月 07, 2021
作者:
D
dragondriver
提交者:
yefu.chen
4月 07, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Add unittest to time tick logic in proxy service
Signed-off-by:
N
dragondriver
<
jiquan.long@zilliz.com
>
上级
decc80a5
变更
4
隐藏空白更改
内联
并排
Showing
4 changed file
with
317 addition
and
5 deletion
+317
-5
internal/msgstream/msgstream_mock.go
internal/msgstream/msgstream_mock.go
+65
-0
internal/proxynode/task_scheduler.go
internal/proxynode/task_scheduler.go
+5
-5
internal/proxyservice/timesync_test.go
internal/proxyservice/timesync_test.go
+192
-0
internal/proxyservice/timetick_test.go
internal/proxyservice/timetick_test.go
+55
-0
未找到文件。
internal/msgstream/msgstream_mock.go
0 → 100644
浏览文件 @
bfb7ca0d
package
msgstream
import
"sync"
type
SimpleMsgStream
struct
{
msgChan
chan
*
MsgPack
msgCount
int
msgCountMtx
sync
.
RWMutex
}
func
(
ms
*
SimpleMsgStream
)
Start
()
{
}
func
(
ms
*
SimpleMsgStream
)
Close
()
{
}
func
(
ms
*
SimpleMsgStream
)
Chan
()
<-
chan
*
MsgPack
{
return
ms
.
msgChan
}
func
(
ms
*
SimpleMsgStream
)
AsProducer
(
channels
[]
string
)
{
}
func
(
ms
*
SimpleMsgStream
)
AsConsumer
(
channels
[]
string
,
subName
string
)
{
}
func
(
ms
*
SimpleMsgStream
)
SetRepackFunc
(
repackFunc
RepackFunc
)
{
}
func
(
ms
*
SimpleMsgStream
)
Produce
(
pack
*
MsgPack
)
error
{
ms
.
msgCountMtx
.
Lock
()
defer
ms
.
msgCountMtx
.
Unlock
()
ms
.
msgChan
<-
pack
ms
.
msgCount
++
return
nil
}
func
(
ms
*
SimpleMsgStream
)
Broadcast
(
pack
*
MsgPack
)
error
{
return
nil
}
func
(
ms
*
SimpleMsgStream
)
Consume
()
*
MsgPack
{
ms
.
msgCountMtx
.
RLock
()
defer
ms
.
msgCountMtx
.
RUnlock
()
if
ms
.
msgCount
<=
0
{
return
nil
}
return
<-
ms
.
msgChan
}
func
(
ms
*
SimpleMsgStream
)
Seek
(
offset
*
MsgPosition
)
error
{
return
nil
}
func
NewSimpleMsgStream
()
*
SimpleMsgStream
{
return
&
SimpleMsgStream
{
msgChan
:
make
(
chan
*
MsgPack
,
1024
),
msgCount
:
0
,
}
}
internal/proxynode/task_scheduler.go
浏览文件 @
bfb7ca0d
...
@@ -20,7 +20,7 @@ import (
...
@@ -20,7 +20,7 @@ import (
type
TaskQueue
interface
{
type
TaskQueue
interface
{
utChan
()
<-
chan
int
utChan
()
<-
chan
int
UT
Empty
()
bool
ut
Empty
()
bool
utFull
()
bool
utFull
()
bool
addUnissuedTask
(
t
task
)
error
addUnissuedTask
(
t
task
)
error
FrontUnissuedTask
()
task
FrontUnissuedTask
()
task
...
@@ -50,7 +50,7 @@ func (queue *BaseTaskQueue) utChan() <-chan int {
...
@@ -50,7 +50,7 @@ func (queue *BaseTaskQueue) utChan() <-chan int {
return
queue
.
utBufChan
return
queue
.
utBufChan
}
}
func
(
queue
*
BaseTaskQueue
)
UT
Empty
()
bool
{
func
(
queue
*
BaseTaskQueue
)
ut
Empty
()
bool
{
queue
.
utLock
.
Lock
()
queue
.
utLock
.
Lock
()
defer
queue
.
utLock
.
Unlock
()
defer
queue
.
utLock
.
Unlock
()
return
queue
.
unissuedTasks
.
Len
()
==
0
return
queue
.
unissuedTasks
.
Len
()
==
0
...
@@ -343,7 +343,7 @@ func (sched *TaskScheduler) definitionLoop() {
...
@@ -343,7 +343,7 @@ func (sched *TaskScheduler) definitionLoop() {
case
<-
sched
.
ctx
.
Done
()
:
case
<-
sched
.
ctx
.
Done
()
:
return
return
case
<-
sched
.
DdQueue
.
utChan
()
:
case
<-
sched
.
DdQueue
.
utChan
()
:
if
!
sched
.
DdQueue
.
UT
Empty
()
{
if
!
sched
.
DdQueue
.
ut
Empty
()
{
t
:=
sched
.
scheduleDdTask
()
t
:=
sched
.
scheduleDdTask
()
sched
.
processTask
(
t
,
sched
.
DdQueue
)
sched
.
processTask
(
t
,
sched
.
DdQueue
)
}
}
...
@@ -358,7 +358,7 @@ func (sched *TaskScheduler) manipulationLoop() {
...
@@ -358,7 +358,7 @@ func (sched *TaskScheduler) manipulationLoop() {
case
<-
sched
.
ctx
.
Done
()
:
case
<-
sched
.
ctx
.
Done
()
:
return
return
case
<-
sched
.
DmQueue
.
utChan
()
:
case
<-
sched
.
DmQueue
.
utChan
()
:
if
!
sched
.
DmQueue
.
UT
Empty
()
{
if
!
sched
.
DmQueue
.
ut
Empty
()
{
t
:=
sched
.
scheduleDmTask
()
t
:=
sched
.
scheduleDmTask
()
go
sched
.
processTask
(
t
,
sched
.
DmQueue
)
go
sched
.
processTask
(
t
,
sched
.
DmQueue
)
}
}
...
@@ -374,7 +374,7 @@ func (sched *TaskScheduler) queryLoop() {
...
@@ -374,7 +374,7 @@ func (sched *TaskScheduler) queryLoop() {
case
<-
sched
.
ctx
.
Done
()
:
case
<-
sched
.
ctx
.
Done
()
:
return
return
case
<-
sched
.
DqQueue
.
utChan
()
:
case
<-
sched
.
DqQueue
.
utChan
()
:
if
!
sched
.
DqQueue
.
UT
Empty
()
{
if
!
sched
.
DqQueue
.
ut
Empty
()
{
t
:=
sched
.
scheduleDqTask
()
t
:=
sched
.
scheduleDqTask
()
go
sched
.
processTask
(
t
,
sched
.
DqQueue
)
go
sched
.
processTask
(
t
,
sched
.
DqQueue
)
}
else
{
}
else
{
...
...
internal/proxyservice/timesync_test.go
0 → 100644
浏览文件 @
bfb7ca0d
package
proxyservice
import
(
"context"
"math"
"testing"
"time"
"github.com/zilliztech/milvus-distributed/internal/log"
"go.uber.org/zap"
"github.com/stretchr/testify/assert"
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
"github.com/zilliztech/milvus-distributed/internal/msgstream"
)
func
ttStreamProduceLoop
(
ctx
context
.
Context
,
ttStream
msgstream
.
MsgStream
,
durationInterval
time
.
Duration
,
sourceID
int64
)
{
log
.
Debug
(
"ttStreamProduceLoop"
,
zap
.
Any
(
"durationInterval"
,
durationInterval
))
timer
:=
time
.
NewTicker
(
durationInterval
)
go
func
()
{
for
{
select
{
case
<-
ctx
.
Done
()
:
return
case
<-
timer
.
C
:
ttMsgs
:=
&
msgstream
.
MsgPack
{
BeginTs
:
0
,
EndTs
:
0
,
Msgs
:
nil
,
StartPositions
:
nil
,
EndPositions
:
nil
,
}
currentT
:=
uint64
(
time
.
Now
()
.
Nanosecond
())
msg
:=
&
msgstream
.
TimeTickMsg
{
BaseMsg
:
msgstream
.
BaseMsg
{
Ctx
:
ctx
,
BeginTimestamp
:
0
,
EndTimestamp
:
0
,
HashValues
:
nil
,
MsgPosition
:
nil
,
},
TimeTickMsg
:
internalpb
.
TimeTickMsg
{
Base
:
&
commonpb
.
MsgBase
{
MsgType
:
0
,
MsgID
:
0
,
Timestamp
:
currentT
,
SourceID
:
sourceID
,
},
},
}
ttMsgs
.
Msgs
=
append
(
ttMsgs
.
Msgs
,
msg
)
_
=
ttStream
.
Produce
(
ttMsgs
)
//log.Debug("ttStreamProduceLoop", zap.Any("Send", currentT))
}
}
}()
}
func
TestSoftTimeTickBarrier_Start
(
t
*
testing
.
T
)
{
ctx
,
cancel
:=
context
.
WithCancel
(
context
.
Background
())
defer
cancel
()
ttStream
:=
msgstream
.
NewSimpleMsgStream
()
sourceID
:=
1
peerIds
:=
[]
UniqueID
{
UniqueID
(
sourceID
)}
interval
:=
100
minTtInterval
:=
Timestamp
(
interval
)
durationInterval
:=
time
.
Duration
(
interval
*
int
(
math
.
Pow10
(
6
)))
>>
18
ttStreamProduceLoop
(
ctx
,
ttStream
,
durationInterval
,
int64
(
sourceID
))
ttBarrier
:=
newSoftTimeTickBarrier
(
ctx
,
ttStream
,
peerIds
,
minTtInterval
)
err
:=
ttBarrier
.
Start
()
assert
.
Equal
(
t
,
nil
,
err
)
defer
ttBarrier
.
Close
()
}
func
TestSoftTimeTickBarrier_Close
(
t
*
testing
.
T
)
{
ctx
,
cancel
:=
context
.
WithCancel
(
context
.
Background
())
defer
cancel
()
ttStream
:=
msgstream
.
NewSimpleMsgStream
()
sourceID
:=
1
peerIds
:=
[]
UniqueID
{
UniqueID
(
sourceID
)}
interval
:=
100
minTtInterval
:=
Timestamp
(
interval
)
durationInterval
:=
time
.
Duration
(
interval
*
int
(
math
.
Pow10
(
6
)))
>>
18
ttStreamProduceLoop
(
ctx
,
ttStream
,
durationInterval
,
int64
(
sourceID
))
ttBarrier
:=
newSoftTimeTickBarrier
(
ctx
,
ttStream
,
peerIds
,
minTtInterval
)
err
:=
ttBarrier
.
Start
()
assert
.
Equal
(
t
,
nil
,
err
)
defer
ttBarrier
.
Close
()
}
func
TestSoftTimeTickBarrier_GetTimeTick
(
t
*
testing
.
T
)
{
ctx
,
cancel
:=
context
.
WithCancel
(
context
.
Background
())
defer
cancel
()
ttStream
:=
msgstream
.
NewSimpleMsgStream
()
sourceID
:=
1
peerIds
:=
[]
UniqueID
{
UniqueID
(
sourceID
)}
interval
:=
100
minTtInterval
:=
Timestamp
(
interval
)
durationInterval
:=
time
.
Duration
(
interval
*
int
(
math
.
Pow10
(
6
)))
>>
18
ttStreamProduceLoop
(
ctx
,
ttStream
,
durationInterval
,
int64
(
sourceID
))
ttBarrier
:=
newSoftTimeTickBarrier
(
ctx
,
ttStream
,
peerIds
,
minTtInterval
)
err
:=
ttBarrier
.
Start
()
assert
.
Equal
(
t
,
nil
,
err
)
defer
ttBarrier
.
Close
()
num
:=
10
for
i
:=
0
;
i
<
num
;
i
++
{
tick
,
err
:=
ttBarrier
.
GetTimeTick
()
assert
.
Equal
(
t
,
nil
,
err
)
log
.
Debug
(
"TestSoftTimeTickBarrier"
,
zap
.
Any
(
"GetTimeTick"
,
tick
))
}
}
func
TestSoftTimeTickBarrier_AddPeer
(
t
*
testing
.
T
)
{
ctx
,
cancel
:=
context
.
WithCancel
(
context
.
Background
())
defer
cancel
()
ttStream
:=
msgstream
.
NewSimpleMsgStream
()
sourceID
:=
1
peerIds
:=
[]
UniqueID
{
UniqueID
(
sourceID
)}
interval
:=
100
minTtInterval
:=
Timestamp
(
interval
)
durationInterval
:=
time
.
Duration
(
interval
*
int
(
math
.
Pow10
(
6
)))
>>
18
ttStreamProduceLoop
(
ctx
,
ttStream
,
durationInterval
,
int64
(
sourceID
))
ttBarrier
:=
newSoftTimeTickBarrier
(
ctx
,
ttStream
,
peerIds
,
minTtInterval
)
err
:=
ttBarrier
.
Start
()
assert
.
Equal
(
t
,
nil
,
err
)
defer
ttBarrier
.
Close
()
newSourceID
:=
UniqueID
(
2
)
err
=
ttBarrier
.
AddPeer
(
newSourceID
)
assert
.
Equal
(
t
,
nil
,
err
)
ttStreamProduceLoop
(
ctx
,
ttStream
,
durationInterval
,
newSourceID
)
num
:=
10
for
i
:=
0
;
i
<
num
;
i
++
{
tick
,
err
:=
ttBarrier
.
GetTimeTick
()
assert
.
Equal
(
t
,
nil
,
err
)
log
.
Debug
(
"TestSoftTimeTickBarrier"
,
zap
.
Any
(
"GetTimeTick"
,
tick
))
}
}
func
TestSoftTimeTickBarrier_TickChan
(
t
*
testing
.
T
)
{
ctx
,
cancel
:=
context
.
WithCancel
(
context
.
Background
())
defer
cancel
()
ttStream
:=
msgstream
.
NewSimpleMsgStream
()
sourceID
:=
1
peerIds
:=
[]
UniqueID
{
UniqueID
(
sourceID
)}
interval
:=
100
minTtInterval
:=
Timestamp
(
interval
)
durationInterval
:=
time
.
Duration
(
interval
*
int
(
math
.
Pow10
(
6
)))
>>
18
ttStreamProduceLoop
(
ctx
,
ttStream
,
durationInterval
,
int64
(
sourceID
))
ttBarrier
:=
newSoftTimeTickBarrier
(
ctx
,
ttStream
,
peerIds
,
minTtInterval
)
err
:=
ttBarrier
.
Start
()
assert
.
Equal
(
t
,
nil
,
err
)
defer
ttBarrier
.
Close
()
duration
:=
time
.
Second
timer
:=
time
.
NewTimer
(
duration
)
for
{
select
{
case
<-
ctx
.
Done
()
:
return
case
<-
timer
.
C
:
return
case
ts
:=
<-
ttBarrier
.
TickChan
()
:
log
.
Debug
(
"TestSoftTimeTickBarrier"
,
zap
.
Any
(
"GetTimeTick"
,
ts
))
}
}
}
internal/proxyservice/timetick_test.go
0 → 100644
浏览文件 @
bfb7ca0d
package
proxyservice
import
(
"context"
"math"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/zilliztech/milvus-distributed/internal/msgstream"
)
func
TestTimeTick_Start
(
t
*
testing
.
T
)
{
ctx
,
cancel
:=
context
.
WithCancel
(
context
.
Background
())
defer
cancel
()
ttStream
:=
msgstream
.
NewSimpleMsgStream
()
sourceID
:=
1
peerIds
:=
[]
UniqueID
{
UniqueID
(
sourceID
)}
interval
:=
100
minTtInterval
:=
Timestamp
(
interval
)
durationInterval
:=
time
.
Duration
(
interval
*
int
(
math
.
Pow10
(
6
)))
>>
18
ttStreamProduceLoop
(
ctx
,
ttStream
,
durationInterval
,
int64
(
sourceID
))
ttBarrier
:=
newSoftTimeTickBarrier
(
ctx
,
ttStream
,
peerIds
,
minTtInterval
)
channels
:=
msgstream
.
NewSimpleMsgStream
()
tick
:=
newTimeTick
(
ctx
,
ttBarrier
,
channels
)
err
:=
tick
.
Start
()
assert
.
Equal
(
t
,
nil
,
err
)
defer
tick
.
Close
()
}
func
TestTimeTick_Close
(
t
*
testing
.
T
)
{
ctx
,
cancel
:=
context
.
WithCancel
(
context
.
Background
())
defer
cancel
()
ttStream
:=
msgstream
.
NewSimpleMsgStream
()
sourceID
:=
1
peerIds
:=
[]
UniqueID
{
UniqueID
(
sourceID
)}
interval
:=
100
minTtInterval
:=
Timestamp
(
interval
)
durationInterval
:=
time
.
Duration
(
interval
*
int
(
math
.
Pow10
(
6
)))
>>
18
ttStreamProduceLoop
(
ctx
,
ttStream
,
durationInterval
,
int64
(
sourceID
))
ttBarrier
:=
newSoftTimeTickBarrier
(
ctx
,
ttStream
,
peerIds
,
minTtInterval
)
channels
:=
msgstream
.
NewSimpleMsgStream
()
tick
:=
newTimeTick
(
ctx
,
ttBarrier
,
channels
)
err
:=
tick
.
Start
()
assert
.
Equal
(
t
,
nil
,
err
)
defer
tick
.
Close
()
}
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录