Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
d3c503f3
M
milvus
项目概览
milvus
/
milvus
11 个月 前同步成功
通知
260
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
前往新版Gitcode,体验更适合开发者的 AI 搜索 >>
未验证
提交
d3c503f3
编写于
6月 28, 2021
作者:
D
dragondriver
提交者:
GitHub
6月 28, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Calculate the real topk in proxy (#6132)
Signed-off-by:
N
dragondriver
<
jiquan.long@zilliz.com
>
上级
6f4ad331
变更
1
隐藏空白更改
内联
并排
Showing
1 changed file
with
22 addition
and
8 deletion
+22
-8
internal/proxy/task.go
internal/proxy/task.go
+22
-8
未找到文件。
internal/proxy/task.go
浏览文件 @
d3c503f3
...
...
@@ -1568,7 +1568,7 @@ func reduceSearchResultsParallel(hits [][]*milvuspb.Hits, nq, availableQueryNode
return
ret
}
func
reduceSearchResultDataParallel
(
searchResultData
[]
*
schemapb
.
SearchResultData
,
nq
,
availableQueryNodeNum
,
topk
int
,
metricType
string
,
maxParallel
int
)
*
milvuspb
.
SearchResults
{
func
reduceSearchResultDataParallel
(
searchResultData
[]
*
schemapb
.
SearchResultData
,
nq
,
availableQueryNodeNum
,
topk
int
,
metricType
string
,
maxParallel
int
)
(
*
milvuspb
.
SearchResults
,
error
)
{
log
.
Debug
(
"reduceSearchResultDataParallel"
,
zap
.
Any
(
"NumOfGoRoutines"
,
maxParallel
))
ret
:=
&
milvuspb
.
SearchResults
{
...
...
@@ -1593,10 +1593,12 @@ func reduceSearchResultDataParallel(searchResultData []*schemapb.SearchResultDat
const
minFloat32
=
-
1
*
float32
(
math
.
MaxFloat32
)
// TODO(yukun): Use parallel function
realTopK
:=
-
1
for
idx
:=
0
;
idx
<
nq
;
idx
++
{
locs
:=
make
([]
int
,
availableQueryNodeNum
)
for
j
:=
0
;
j
<
topk
;
j
++
{
j
:=
0
for
;
j
<
topk
;
j
++
{
valid
:=
false
choice
,
maxDistance
:=
0
,
minFloat32
for
q
,
loc
:=
range
locs
{
// query num, the number of ways to merge
...
...
@@ -1696,7 +1698,7 @@ func reduceSearchResultDataParallel(searchResultData []*schemapb.SearchResultDat
}
default
:
log
.
Debug
(
"Not supported field type"
)
return
nil
return
nil
,
fmt
.
Errorf
(
"not supported field type: %s"
,
fieldData
.
Type
.
String
())
}
case
*
schemapb
.
FieldData_Vectors
:
dim
:=
fieldType
.
Vectors
.
Dim
...
...
@@ -1729,9 +1731,15 @@ func reduceSearchResultDataParallel(searchResultData []*schemapb.SearchResultDat
ret
.
Results
.
Scores
=
append
(
ret
.
Results
.
Scores
,
searchResultData
[
choice
]
.
Scores
[
idx
*
topk
+
choiceOffset
])
locs
[
choice
]
++
}
if
realTopK
!=
-
1
&&
realTopK
!=
j
{
log
.
Warn
(
"Proxy Reduce Search Result"
,
zap
.
Error
(
errors
.
New
(
"the length (topk) between all result of query is different"
)))
// return nil, errors.New("the length (topk) between all result of query is different")
}
realTopK
=
j
}
ret
.
Results
.
TopK
=
int64
(
realTopK
)
if
metricType
!=
"IP"
{
for
k
:=
range
ret
.
Results
.
Scores
{
ret
.
Results
.
Scores
[
k
]
*=
-
1
...
...
@@ -1742,7 +1750,7 @@ func reduceSearchResultDataParallel(searchResultData []*schemapb.SearchResultDat
// return nil
// }
return
ret
return
ret
,
nil
}
func
reduceSearchResultsSerial
(
hits
[][]
*
milvuspb
.
Hits
,
nq
,
availableQueryNodeNum
,
topk
int
,
metricType
string
)
*
milvuspb
.
SearchResults
{
...
...
@@ -1767,7 +1775,7 @@ func reduceSearchResults(hits [][]*milvuspb.Hits, nq, availableQueryNodeNum, top
return
reduceSearchResultsParallelByCPU
(
hits
,
nq
,
availableQueryNodeNum
,
topk
,
metricType
)
}
func
reduceSearchResultData
(
searchResultData
[]
*
schemapb
.
SearchResultData
,
nq
,
availableQueryNodeNum
,
topk
int
,
metricType
string
)
*
milvuspb
.
SearchResults
{
func
reduceSearchResultData
(
searchResultData
[]
*
schemapb
.
SearchResultData
,
nq
,
availableQueryNodeNum
,
topk
int
,
metricType
string
)
(
*
milvuspb
.
SearchResults
,
error
)
{
t
:=
time
.
Now
()
defer
func
()
{
log
.
Debug
(
"reduceSearchResults"
,
zap
.
Any
(
"time cost"
,
time
.
Since
(
t
)))
...
...
@@ -1853,7 +1861,10 @@ func (st *SearchTask) PostExecute(ctx context.Context) error {
}
nq
:=
results
[
0
]
.
NumQueries
topk
:=
results
[
0
]
.
TopK
topk
:=
0
for
_
,
partialResult
:=
range
results
{
topk
=
getMax
(
topk
,
int
(
partialResult
.
TopK
))
}
if
nq
<=
0
{
st
.
result
=
&
milvuspb
.
SearchResults
{
Status
:
&
commonpb
.
Status
{
...
...
@@ -1864,7 +1875,10 @@ func (st *SearchTask) PostExecute(ctx context.Context) error {
return
nil
}
st
.
result
=
reduceSearchResultData
(
results
,
int
(
nq
),
availableQueryNodeNum
,
int
(
topk
),
searchResults
[
0
]
.
MetricType
)
st
.
result
,
err
=
reduceSearchResultData
(
results
,
int
(
nq
),
availableQueryNodeNum
,
topk
,
searchResults
[
0
]
.
MetricType
)
if
err
!=
nil
{
return
err
}
schema
,
err
:=
globalMetaCache
.
GetCollectionSchema
(
ctx
,
st
.
query
.
CollectionName
)
if
err
!=
nil
{
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录