Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
649732c1
M
milvus
项目概览
milvus
/
milvus
大约 1 年 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
649732c1
编写于
9月 25, 2020
作者:
B
bigsheeper
提交者:
yefu.chen
9月 25, 2020
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Fix search error about search time
Signed-off-by:
N
bigsheeper
<
yihao.dai@zilliz.com
>
上级
3bd6c541
变更
9
隐藏空白更改
内联
并排
Showing
9 changed file
with
76 addition
and
27 deletion
+76
-27
pkg/master/informer/pulsar.go
pkg/master/informer/pulsar.go
+4
-1
pkg/master/mock/segment.go
pkg/master/mock/segment.go
+13
-3
pkg/master/server.go
pkg/master/server.go
+23
-9
proxy/src/message_client/ClientV2.cpp
proxy/src/message_client/ClientV2.cpp
+5
-0
proxy/src/server/delivery/ReqScheduler.cpp
proxy/src/server/delivery/ReqScheduler.cpp
+4
-1
reader/message_client/message_client.go
reader/message_client/message_client.go
+2
-1
reader/read_node/query_node.go
reader/read_node/query_node.go
+21
-8
reader/read_node/query_node_time.go
reader/read_node/query_node_time.go
+2
-2
reader/read_node/segment_service.go
reader/read_node/segment_service.go
+2
-2
未找到文件。
pkg/master/informer/pulsar.go
浏览文件 @
649732c1
...
...
@@ -49,7 +49,10 @@ func (pc PulsarClient) Listener(ssChan chan mock.SegmentStats) error {
if
err
!=
nil
{
log
.
Fatal
(
err
)
}
m
,
_
:=
mock
.
SegmentUnMarshal
(
msg
.
Payload
())
m
,
err
:=
mock
.
SegmentUnMarshal
(
msg
.
Payload
())
if
err
!=
nil
{
log
.
Println
(
"SegmentUnMarshal Failed"
)
}
fmt
.
Printf
(
"Received message msgId: %#v -- content: '%s'
\n
"
,
msg
.
ID
(),
m
.
SegementID
)
ssChan
<-
m
...
...
pkg/master/mock/segment.go
浏览文件 @
649732c1
...
...
@@ -3,6 +3,7 @@ package mock
import
(
"bytes"
"encoding/gob"
"github.com/golang/protobuf/proto"
"time"
masterpb
"github.com/czs007/suvlim/pkg/master/grpc/master"
...
...
@@ -16,6 +17,9 @@ type SegmentStats struct {
Rows
int64
}
// map[SegmentID]SegmentCloseTime
type
SegmentCloseLog
map
[
uint64
]
uint64
func
SegmentMarshal
(
s
SegmentStats
)
([]
byte
,
error
)
{
var
nb
bytes
.
Buffer
enc
:=
gob
.
NewEncoder
(
&
nb
)
...
...
@@ -27,12 +31,18 @@ func SegmentMarshal(s SegmentStats) ([]byte, error) {
}
func
SegmentUnMarshal
(
data
[]
byte
)
(
SegmentStats
,
error
)
{
var
ss
SegmentStats
dec
:=
gob
.
NewDecoder
(
bytes
.
NewBuffer
(
data
))
err
:=
dec
.
Decode
(
&
ss
)
var
pbSS
masterpb
.
SegmentStat
err
:=
proto
.
Unmarshal
(
data
,
&
pbSS
)
if
err
!=
nil
{
return
SegmentStats
{},
err
}
var
ss
=
SegmentStats
{
SegementID
:
pbSS
.
SegmentId
,
MemorySize
:
pbSS
.
MemorySize
,
MemoryRate
:
float64
(
pbSS
.
MemoryRate
),
Status
:
pbSS
.
Status
,
Rows
:
pbSS
.
Rows
,
}
return
ss
,
nil
}
...
...
pkg/master/server.go
浏览文件 @
649732c1
...
...
@@ -21,7 +21,7 @@ import (
)
func
Run
()
{
go
mock
.
FakePulsarProducer
()
//
go mock.FakePulsarProducer()
go
SegmentStatsController
()
collectionChan
:=
make
(
chan
*
messagepb
.
Mapping
)
defer
close
(
collectionChan
)
...
...
@@ -45,27 +45,39 @@ func SegmentStatsController() {
ssChan
:=
make
(
chan
mock
.
SegmentStats
,
10
)
defer
close
(
ssChan
)
ssClient
:=
informer
.
NewPulsarClient
()
segmentCloseLog
:=
make
(
map
[
uint64
]
uint64
,
0
)
go
ssClient
.
Listener
(
ssChan
)
for
{
select
{
case
ss
:=
<-
ssChan
:
ComputeCloseTime
(
ss
,
kvbase
)
ComputeCloseTime
(
&
segmentCloseLog
,
ss
,
kvbase
)
UpdateSegmentStatus
(
ss
,
kvbase
)
case
<-
time
.
After
(
5
*
time
.
Second
)
:
fmt
.
Println
(
"timeout"
)
return
//
case <-time.After(5 * time.Second):
//
fmt.Println("timeout")
//
return
}
}
}
func
ComputeCloseTime
(
ss
mock
.
SegmentStats
,
kvbase
kv
.
Base
)
error
{
func
ComputeCloseTime
(
segmentCloseLog
*
map
[
uint64
]
uint64
,
ss
mock
.
SegmentStats
,
kvbase
kv
.
Base
)
error
{
segmentID
:=
ss
.
SegementID
if
_
,
ok
:=
(
*
segmentCloseLog
)[
segmentID
];
ok
{
// This segment has been closed
log
.
Println
(
"Segment"
,
segmentID
,
"has been closed"
)
return
nil
}
if
int
(
ss
.
MemorySize
)
>
int
(
conf
.
Config
.
Master
.
SegmentThreshole
*
0.8
)
{
currentTime
:=
time
.
Now
()
memRate
:=
int
(
ss
.
MemoryRate
)
if
memRate
==
0
{
memRate
=
1
//memRate = 1
log
.
Println
(
"memRate = 0"
)
return
nil
}
sec
:=
int
(
conf
.
Config
.
Master
.
SegmentThreshole
*
0.2
)
/
memRate
sec
:=
float64
(
conf
.
Config
.
Master
.
SegmentThreshole
*
0.2
)
/
float64
(
memRate
)
data
,
err
:=
kvbase
.
Load
(
"segment/"
+
strconv
.
Itoa
(
int
(
ss
.
SegementID
)))
if
err
!=
nil
{
return
err
...
...
@@ -74,13 +86,15 @@ func ComputeCloseTime(ss mock.SegmentStats, kvbase kv.Base) error {
if
err
!=
nil
{
return
err
}
seg
.
CloseTimeStamp
=
uint64
(
currentTime
.
Add
(
time
.
Duration
(
sec
)
*
time
.
Second
)
.
Unix
())
segmentLogicTime
:=
seg
.
CloseTimeStamp
<<
46
>>
46
seg
.
CloseTimeStamp
=
uint64
(
currentTime
.
Add
(
time
.
Duration
(
sec
)
*
time
.
Second
)
.
Unix
())
<<
18
+
segmentLogicTime
fmt
.
Println
(
"memRate = "
,
memRate
,
",sec = "
,
sec
,
",Close time = "
,
seg
.
CloseTimeStamp
)
updateData
,
err
:=
mock
.
Segment2JSON
(
*
seg
)
if
err
!=
nil
{
return
err
}
kvbase
.
Save
(
"segment/"
+
strconv
.
Itoa
(
int
(
ss
.
SegementID
)),
updateData
)
(
*
segmentCloseLog
)[
segmentID
]
=
seg
.
CloseTimeStamp
//create new segment
newSegID
:=
id
.
New
()
.
Uint64
()
newSeg
:=
mock
.
NewSegment
(
newSegID
,
seg
.
CollectionID
,
seg
.
CollectionName
,
"default"
,
seg
.
ChannelStart
,
seg
.
ChannelEnd
,
currentTime
,
time
.
Unix
(
1
<<
36
-
1
,
0
))
...
...
proxy/src/message_client/ClientV2.cpp
浏览文件 @
649732c1
...
...
@@ -146,9 +146,14 @@ Status MsgClientV2::GetQueryResult(int64_t query_id, milvus::grpc::QueryResult *
auto
status
=
search_res_msg
.
ParseFromString
(
msg
.
getDataAsString
());
if
(
status
)
{
auto
message
=
std
::
make_shared
<
grpc
::
QueryResult
>
(
search_res_msg
);
if
(
message
->
status
().
error_code
()
!=
grpc
::
ErrorCode
::
SUCCESS
)
{
consumer_
->
acknowledge
(
msg
);
return
Status
(
DB_ERROR
,
"Search Failed"
);
}
total_results
[
message
->
query_id
()].
push_back
(
message
);
consumer_
->
acknowledge
(
msg
);
}
else
{
consumer_
->
acknowledge
(
msg
);
return
Status
(
DB_ERROR
,
"can't parse message which from pulsar!"
);
}
}
...
...
proxy/src/server/delivery/ReqScheduler.cpp
浏览文件 @
649732c1
...
...
@@ -121,11 +121,14 @@ ReqScheduler::TakeToExecute(ReqQueuePtr req_queue) {
}
try
{
if
(
req
->
type
()
==
ReqType
::
kInsert
||
req
->
type
()
==
ReqType
::
kDeleteEntityByID
||
req
->
type
()
==
ReqType
::
kSearch
)
{
if
(
req
->
type
()
==
ReqType
::
kInsert
||
req
->
type
()
==
ReqType
::
kDeleteEntityByID
)
{
std
::
lock_guard
lock
(
time_syc_mtx_
);
sending_
=
true
;
req
->
SetTimestamp
(
TSOracle
::
GetInstance
().
GetTimeStamp
());
}
if
(
req
->
type
()
==
ReqType
::
kSearch
){
req
->
SetTimestamp
(
TSOracle
::
GetInstance
().
GetTimeStamp
());
}
auto
status
=
req
->
Execute
();
if
(
!
status
.
ok
())
{
LOG_SERVER_ERROR_
<<
"Req failed with code: "
<<
status
.
ToString
();
...
...
reader/message_client/message_client.go
浏览文件 @
649732c1
...
...
@@ -167,7 +167,8 @@ func (mc *MessageClient) InitClient(url string) {
mc
.
searchResultProducers
[
key
]
=
mc
.
creatProducer
(
topic
)
}
//mc.searchResultProducer = mc.creatProducer("SearchResult")
mc
.
segmentsStatisticProducer
=
mc
.
creatProducer
(
"SegmentsStatistic"
)
SegmentsStatisticTopicName
:=
conf
.
Config
.
Master
.
PulsarTopic
mc
.
segmentsStatisticProducer
=
mc
.
creatProducer
(
SegmentsStatisticTopicName
)
//create consumer
mc
.
searchConsumer
=
mc
.
createConsumer
(
"Search"
)
...
...
reader/read_node/query_node.go
浏览文件 @
649732c1
...
...
@@ -16,6 +16,7 @@ import "C"
import
(
"encoding/json"
"fmt"
"github.com/czs007/suvlim/conf"
"github.com/stretchr/testify/assert"
"log"
"sort"
...
...
@@ -94,7 +95,7 @@ func NewQueryNode(queryNodeId uint64, timeSync uint64) *QueryNode {
ReadTimeSyncMin
:
timeSync
,
ReadTimeSyncMax
:
timeSync
,
WriteTimeSync
:
timeSync
,
Se
arch
TimeSync
:
timeSync
,
Se
rvice
TimeSync
:
timeSync
,
TSOTimeSync
:
timeSync
,
}
...
...
@@ -134,7 +135,7 @@ func CreateQueryNode(queryNodeId uint64, timeSync uint64, mc *message_client.Mes
ReadTimeSyncMin
:
timeSync
,
ReadTimeSyncMax
:
timeSync
,
WriteTimeSync
:
timeSync
,
Se
arch
TimeSync
:
timeSync
,
Se
rvice
TimeSync
:
timeSync
,
TSOTimeSync
:
timeSync
,
}
...
...
@@ -327,8 +328,8 @@ func (node *QueryNode) TestInsertDelete(timeRange TimeRange) {
func
(
node
*
QueryNode
)
RunSearch
(
wg
*
sync
.
WaitGroup
)
{
for
{
msg
,
ok
:=
<-
node
.
messageClient
.
GetSearchChan
()
if
ok
{
select
{
case
msg
:=
<-
node
.
messageClient
.
GetSearchChan
()
:
node
.
messageClient
.
SearchMsg
=
node
.
messageClient
.
SearchMsg
[
:
0
]
node
.
messageClient
.
SearchMsg
=
append
(
node
.
messageClient
.
SearchMsg
,
msg
)
fmt
.
Println
(
"Do Search..."
)
...
...
@@ -337,6 +338,8 @@ func (node *QueryNode) RunSearch(wg *sync.WaitGroup) {
fmt
.
Println
(
"Search Failed"
)
node
.
PublishFailedSearchResult
()
}
default
:
}
}
wg
.
Done
()
...
...
@@ -569,15 +572,25 @@ func (node *QueryNode) Search(searchMessages []*msgPb.SearchMsg) msgPb.Status {
var
clientId
=
msg
.
ClientId
var
resultsTmp
=
make
([]
SearchResultTmp
,
0
)
var
timestamp
=
msg
.
Timestamp
var
searchTimestamp
=
msg
.
Timestamp
// ServiceTimeSync update by readerTimeSync, which is get from proxy.
// Proxy send this timestamp per `conf.Config.Timesync.Interval` milliseconds.
// However, timestamp of search request (searchTimestamp) is precision time.
// So the ServiceTimeSync is always less than searchTimestamp.
// Here, we manually make searchTimestamp's logic time minus `conf.Config.Timesync.Interval` milliseconds.
// Which means `searchTimestamp.logicTime = searchTimestamp.logicTime - conf.Config.Timesync.Interval`.
var
logicTimestamp
=
searchTimestamp
<<
46
>>
46
searchTimestamp
=
(
searchTimestamp
>>
18
-
uint64
(
conf
.
Config
.
Timesync
.
Interval
))
<<
18
+
logicTimestamp
var
vector
=
msg
.
Records
// We now only the first Json is valid.
var
queryJson
=
msg
.
Json
[
0
]
// 1. Timestamp check
// TODO: return or wait? Or adding graceful time
if
timestamp
>
node
.
queryNodeTimeSync
.
Search
TimeSync
{
fmt
.
Println
(
"Invalid query time, timestamp = "
,
timestamp
,
", SearchTimeSync = "
,
node
.
queryNodeTimeSync
.
Search
TimeSync
)
if
searchTimestamp
>
node
.
queryNodeTimeSync
.
Service
TimeSync
{
fmt
.
Println
(
"Invalid query time, timestamp = "
,
searchTimestamp
,
", SearchTimeSync = "
,
node
.
queryNodeTimeSync
.
Service
TimeSync
)
return
msgPb
.
Status
{
ErrorCode
:
1
}
}
...
...
@@ -586,7 +599,7 @@ func (node *QueryNode) Search(searchMessages []*msgPb.SearchMsg) msgPb.Status {
// 3. Do search in all segments
for
_
,
segment
:=
range
node
.
SegmentsMap
{
var
res
,
err
=
segment
.
SegmentSearch
(
query
,
t
imestamp
,
vector
)
var
res
,
err
=
segment
.
SegmentSearch
(
query
,
searchT
imestamp
,
vector
)
if
err
!=
nil
{
fmt
.
Println
(
err
.
Error
())
return
msgPb
.
Status
{
ErrorCode
:
1
}
...
...
reader/read_node/query_node_time.go
浏览文件 @
649732c1
...
...
@@ -4,7 +4,7 @@ type QueryNodeTime struct {
ReadTimeSyncMin
uint64
ReadTimeSyncMax
uint64
WriteTimeSync
uint64
Se
arch
TimeSync
uint64
Se
rvice
TimeSync
uint64
TSOTimeSync
uint64
}
...
...
@@ -25,7 +25,7 @@ func (t *QueryNodeTime) UpdateWriteTimeSync() {
}
func
(
t
*
QueryNodeTime
)
UpdateSearchTimeSync
(
timeRange
TimeRange
)
{
t
.
Se
arch
TimeSync
=
timeRange
.
timestampMax
t
.
Se
rvice
TimeSync
=
timeRange
.
timestampMax
}
func
(
t
*
QueryNodeTime
)
UpdateTSOTimeSync
()
{
...
...
reader/read_node/segment_service.go
浏览文件 @
649732c1
...
...
@@ -80,8 +80,8 @@ func (node *QueryNode) SegmentStatistic(sleepMillisecondTime int) {
statisticData
=
append
(
statisticData
,
stat
)
}
fmt
.
Println
(
"Publish segment statistic"
)
fmt
.
Println
(
statisticData
)
//
fmt.Println("Publish segment statistic")
//
fmt.Println(statisticData)
var
status
=
node
.
PublicStatistic
(
&
statisticData
)
if
status
.
ErrorCode
!=
msgPb
.
ErrorCode_SUCCESS
{
log
.
Printf
(
"Publish segments statistic failed"
)
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录