Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
0ab70271
M
milvus
项目概览
milvus
/
milvus
大约 1 年 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
0ab70271
编写于
8月 24, 2021
作者:
C
Cai Yudong
提交者:
GitHub
8月 24, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
correct empty search result handle (#7244)
Signed-off-by:
N
yudong.cai
<
yudong.cai@zilliz.com
>
上级
e782ee94
变更
4
展开全部
显示空白变更内容
内联
并排
Showing
4 changed file
with
205 addition
and
214 deletion
+205
-214
internal/proto/internal.proto
internal/proto/internal.proto
+8
-10
internal/proto/internalpb/internal.pb.go
internal/proto/internalpb/internal.pb.go
+164
-154
internal/proxy/task.go
internal/proxy/task.go
+25
-20
internal/querynode/query_collection.go
internal/querynode/query_collection.go
+8
-30
未找到文件。
internal/proto/internal.proto
浏览文件 @
0ab70271
...
...
@@ -144,17 +144,15 @@ message SearchResults {
common.Status
status
=
2
;
string
result_channelID
=
3
;
string
metric_type
=
4
;
repeated
bytes
hits
=
5
;
int64
num_queries
=
5
;
int64
top_k
=
6
;
repeated
int64
sealed_segmentIDs_searched
=
7
;
repeated
string
channelIDs_searched
=
8
;
repeated
int64
global_sealed_segmentIDs
=
9
;
// schema.SearchResultsData inside
bytes
sliced_blob
=
9
;
int64
sliced_num_count
=
10
;
int64
sliced_offset
=
11
;
repeated
int64
sealed_segmentIDs_searched
=
6
;
repeated
string
channelIDs_searched
=
7
;
repeated
int64
global_sealed_segmentIDs
=
8
;
bytes
sliced_blob
=
10
;
int64
sliced_num_count
=
11
;
int64
sliced_offset
=
12
;
}
message
RetrieveRequest
{
...
...
internal/proto/internalpb/internal.pb.go
浏览文件 @
0ab70271
此差异已折叠。
点击以展开。
internal/proxy/task.go
浏览文件 @
0ab70271
...
...
@@ -1659,9 +1659,8 @@ func decodeSearchResults(searchResults []*internalpb.SearchResults) ([]*schemapb
// return decodeSearchResultsParallelByCPU(searchResults)
}
func
reduceSearchResultDataParallel
(
searchResultData
[]
*
schemapb
.
SearchResultData
,
availableQueryNodeNum
int64
,
metricType
string
,
maxParallel
int
)
(
*
milvuspb
.
SearchResults
,
error
)
{
nq
:=
searchResultData
[
0
]
.
NumQueries
topk
:=
searchResultData
[
0
]
.
TopK
func
reduceSearchResultDataParallel
(
searchResultData
[]
*
schemapb
.
SearchResultData
,
availableQueryNodeNum
int64
,
nq
int64
,
topk
int64
,
metricType
string
,
maxParallel
int
)
(
*
milvuspb
.
SearchResults
,
error
)
{
log
.
Debug
(
"reduceSearchResultDataParallel"
,
zap
.
Int
(
"len(searchResultData)"
,
len
(
searchResultData
)),
...
...
@@ -1887,25 +1886,26 @@ func reduceSearchResultDataParallel(searchResultData []*schemapb.SearchResultDat
return
ret
,
nil
}
func
reduceSearchResultData
(
searchResultData
[]
*
schemapb
.
SearchResultData
,
availableQueryNodeNum
int64
,
metricType
string
)
(
*
milvuspb
.
SearchResults
,
error
)
{
func
reduceSearchResultData
(
searchResultData
[]
*
schemapb
.
SearchResultData
,
availableQueryNodeNum
int64
,
nq
int64
,
topk
int64
,
metricType
string
)
(
*
milvuspb
.
SearchResults
,
error
)
{
t
:=
time
.
Now
()
defer
func
()
{
log
.
Debug
(
"reduceSearchResults"
,
zap
.
Any
(
"time cost"
,
time
.
Since
(
t
)))
}()
return
reduceSearchResultDataParallel
(
searchResultData
,
availableQueryNodeNum
,
metricType
,
runtime
.
NumCPU
())
return
reduceSearchResultDataParallel
(
searchResultData
,
availableQueryNodeNum
,
nq
,
topk
,
metricType
,
runtime
.
NumCPU
())
}
func
printSearchResult
(
partialSearchResult
*
internalpb
.
SearchResults
)
{
for
i
:=
0
;
i
<
len
(
partialSearchResult
.
Hits
);
i
++
{
testHits
:=
milvuspb
.
Hits
{}
err
:=
proto
.
Unmarshal
(
partialSearchResult
.
Hits
[
i
],
&
testHits
)
if
err
!=
nil
{
panic
(
err
)
}
fmt
.
Println
(
testHits
.
IDs
)
fmt
.
Println
(
testHits
.
Scores
)
}
}
//
func printSearchResult(partialSearchResult *internalpb.SearchResults) {
//
for i := 0; i < len(partialSearchResult.Hits); i++ {
//
testHits := milvuspb.Hits{}
//
err := proto.Unmarshal(partialSearchResult.Hits[i], &testHits)
//
if err != nil {
//
panic(err)
//
}
//
fmt.Println(testHits.IDs)
//
fmt.Println(testHits.Scores)
//
}
//
}
func
(
st
*
SearchTask
)
PostExecute
(
ctx
context
.
Context
)
error
{
t0
:=
time
.
Now
()
...
...
@@ -1947,11 +1947,11 @@ func (st *SearchTask) PostExecute(ctx context.Context) error {
availableQueryNodeNum
=
0
for
_
,
partialSearchResult
:=
range
filterSearchResult
{
if
partialSearchResult
.
SlicedBlob
==
nil
{
filterReason
+=
"nq is zero
\n
"
continue
}
filterReason
+=
"empty search result
\n
"
}
else
{
availableQueryNodeNum
++
}
}
log
.
Debug
(
"Proxy Search PostExecute stage2"
,
zap
.
Any
(
"availableQueryNodeNum"
,
availableQueryNodeNum
))
if
availableQueryNodeNum
<=
0
{
...
...
@@ -1962,6 +1962,10 @@ func (st *SearchTask) PostExecute(ctx context.Context) error {
ErrorCode
:
commonpb
.
ErrorCode_Success
,
Reason
:
filterReason
,
},
Results
:
&
schemapb
.
SearchResultData
{
NumQueries
:
searchResults
[
0
]
.
NumQueries
,
Topks
:
make
([]
int64
,
searchResults
[
0
]
.
NumQueries
),
},
}
return
nil
}
...
...
@@ -1972,7 +1976,8 @@ func (st *SearchTask) PostExecute(ctx context.Context) error {
return
err
}
st
.
result
,
err
=
reduceSearchResultData
(
results
,
int64
(
availableQueryNodeNum
),
searchResults
[
0
]
.
MetricType
)
st
.
result
,
err
=
reduceSearchResultData
(
results
,
int64
(
availableQueryNodeNum
),
searchResults
[
0
]
.
NumQueries
,
searchResults
[
0
]
.
TopK
,
searchResults
[
0
]
.
MetricType
)
if
err
!=
nil
{
return
err
}
...
...
internal/querynode/query_collection.go
浏览文件 @
0ab70271
...
...
@@ -867,31 +867,7 @@ func (q *queryCollection) search(msg queryMsg) error {
sp
.
LogFields
(
oplog
.
String
(
"statistical time"
,
"segment search end"
))
if
len
(
searchResults
)
<=
0
{
for
_
,
group
:=
range
searchRequests
{
nq
:=
group
.
getNumOfQuery
()
nilHits
:=
make
([][]
byte
,
nq
)
hit
:=
&
milvuspb
.
Hits
{}
for
i
:=
0
;
i
<
int
(
nq
);
i
++
{
bs
,
err
:=
proto
.
Marshal
(
hit
)
if
err
!=
nil
{
return
err
}
nilHits
[
i
]
=
bs
}
// TODO: remove inefficient code in cgo and use SearchResultData directly
// TODO: Currently add a translate layer from hits to SearchResultData
// TODO: hits marshal and unmarshal is likely bottleneck
transformed
,
err
:=
translateHits
(
schema
,
searchMsg
.
OutputFieldsId
,
nilHits
)
if
err
!=
nil
{
return
err
}
byteBlobs
,
err
:=
proto
.
Marshal
(
transformed
)
if
err
!=
nil
{
return
err
}
for
range
searchRequests
{
resultChannelInt
:=
0
searchResultMsg
:=
&
msgstream
.
SearchResultMsg
{
BaseMsg
:
msgstream
.
BaseMsg
{
Ctx
:
searchMsg
.
Ctx
,
HashValues
:
[]
uint32
{
uint32
(
resultChannelInt
)}},
...
...
@@ -904,11 +880,12 @@ func (q *queryCollection) search(msg queryMsg) error {
},
Status
:
&
commonpb
.
Status
{
ErrorCode
:
commonpb
.
ErrorCode_Success
},
ResultChannelID
:
searchMsg
.
ResultChannelID
,
Hits
:
nilHits
,
SlicedBlob
:
byteBlobs
,
MetricType
:
plan
.
getMetricType
(),
NumQueries
:
queryNum
,
TopK
:
topK
,
SlicedBlob
:
nil
,
SlicedOffset
:
1
,
SlicedNumCount
:
1
,
MetricType
:
plan
.
getMetricType
(),
SealedSegmentIDsSearched
:
sealedSegmentSearched
,
ChannelIDsSearched
:
q
.
collection
.
getVChannels
(),
GlobalSealedSegmentIDs
:
globalSealedSegments
,
...
...
@@ -995,11 +972,12 @@ func (q *queryCollection) search(msg queryMsg) error {
},
Status
:
&
commonpb
.
Status
{
ErrorCode
:
commonpb
.
ErrorCode_Success
},
ResultChannelID
:
searchMsg
.
ResultChannelID
,
Hits
:
hits
,
MetricType
:
plan
.
getMetricType
(),
NumQueries
:
queryNum
,
TopK
:
topK
,
SlicedBlob
:
byteBlobs
,
SlicedOffset
:
1
,
SlicedNumCount
:
1
,
MetricType
:
plan
.
getMetricType
(),
SealedSegmentIDsSearched
:
sealedSegmentSearched
,
ChannelIDsSearched
:
q
.
collection
.
getVChannels
(),
GlobalSealedSegmentIDs
:
globalSealedSegments
,
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录