Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
24f94112
M
milvus
项目概览
milvus
/
milvus
11 个月 前同步成功
通知
260
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
24f94112
编写于
10月 27, 2021
作者:
C
Cai Yudong
提交者:
GitHub
10月 27, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Optimize queryTask PostExecute (#10739)
Signed-off-by:
N
yudong.cai
<
yudong.cai@zilliz.com
>
上级
f5323de2
变更
1
隐藏空白更改
内联
并排
Showing
1 changed file
with
56 addition
and
66 deletion
+56
-66
internal/proxy/task.go
internal/proxy/task.go
+56
-66
未找到文件。
internal/proxy/task.go
浏览文件 @
24f94112
...
...
@@ -1753,7 +1753,7 @@ func selectSearchResultData(dataArray []*schemapb.SearchResultData, offsets []in
return
sel
}
func
copySearchResultData
(
dst
*
schemapb
.
SearchResultData
,
src
*
schemapb
.
SearchResultData
,
idx
int64
)
error
{
func
copySearchResultData
(
dst
*
schemapb
.
SearchResultData
,
src
*
schemapb
.
SearchResultData
,
idx
int64
)
{
for
i
,
fieldData
:=
range
src
.
FieldsData
{
switch
fieldType
:=
fieldData
.
Field
.
(
type
)
{
case
*
schemapb
.
FieldData_Scalars
:
...
...
@@ -1829,7 +1829,6 @@ func copySearchResultData(dst *schemapb.SearchResultData, src *schemapb.SearchRe
}
default
:
log
.
Debug
(
"Not supported field type"
,
zap
.
String
(
"field type"
,
fieldData
.
Type
.
String
()))
return
fmt
.
Errorf
(
"not supported field type: %s"
,
fieldData
.
Type
.
String
())
}
case
*
schemapb
.
FieldData_Vectors
:
dim
:=
fieldType
.
Vectors
.
Dim
...
...
@@ -1867,11 +1866,9 @@ func copySearchResultData(dst *schemapb.SearchResultData, src *schemapb.SearchRe
}
default
:
log
.
Debug
(
"Not supported field type"
,
zap
.
String
(
"field type"
,
fieldData
.
Type
.
String
()))
return
fmt
.
Errorf
(
"not supported field type: %s"
,
fieldData
.
Type
.
String
())
}
}
}
return
nil
}
//func printSearchResultData(data *schemapb.SearchResultData, header string) {
...
...
@@ -2035,11 +2032,10 @@ func (st *searchTask) PostExecute(ctx context.Context) error {
}
}
availableQueryNodeNum
:=
len
(
filterSearchResults
)
log
.
Debug
(
"Proxy Search PostExecute stage1"
,
zap
.
Any
(
"
availableQueryNodeNum"
,
availableQueryNodeNum
))
zap
.
Any
(
"
len(filterSearchResults)"
,
len
(
filterSearchResults
)
))
tr
.
Record
(
"Proxy Search PostExecute stage1 done"
)
if
availableQueryNodeNum
<=
0
{
if
len
(
filterSearchResults
)
<=
0
{
st
.
result
=
&
milvuspb
.
SearchResults
{
Status
:
&
commonpb
.
Status
{
ErrorCode
:
commonpb
.
ErrorCode_UnexpectedError
,
...
...
@@ -2054,12 +2050,11 @@ func (st *searchTask) PostExecute(ctx context.Context) error {
return
err
}
log
.
Debug
(
"Proxy Search PostExecute stage2"
,
zap
.
Any
(
"availableQueryNodeNum"
,
availableQueryNodeNum
))
log
.
Debug
(
"Proxy Search PostExecute stage2"
,
zap
.
Any
(
"len(validSearchResults)"
,
len
(
validSearchResults
)))
if
len
(
validSearchResults
)
<=
0
{
filterReason
+=
"empty search result
\n
"
log
.
Debug
(
"Proxy Search PostExecute stage2 failed"
,
zap
.
Any
(
"filterReason"
,
filterReason
))
filterReason
+=
"empty search result
\n
"
st
.
result
=
&
milvuspb
.
SearchResults
{
Status
:
&
commonpb
.
Status
{
ErrorCode
:
commonpb
.
ErrorCode_Success
,
...
...
@@ -2420,6 +2415,45 @@ func (qt *queryTask) Execute(ctx context.Context) error {
return
err
}
func
copyQueryResultData
(
dst
*
milvuspb
.
QueryResults
,
src
*
internalpb
.
RetrieveResults
)
{
// handles initialization, cannot use idx==0 since first result may be empty
if
len
(
dst
.
FieldsData
)
==
0
{
dst
.
FieldsData
=
append
(
dst
.
FieldsData
,
src
.
FieldsData
...
)
}
else
{
for
i
,
fieldData
:=
range
src
.
FieldsData
{
switch
fieldType
:=
fieldData
.
Field
.
(
type
)
{
case
*
schemapb
.
FieldData_Scalars
:
dstScalar
:=
dst
.
FieldsData
[
i
]
.
GetScalars
()
switch
srcScalar
:=
fieldType
.
Scalars
.
Data
.
(
type
)
{
case
*
schemapb
.
ScalarField_BoolData
:
dstScalar
.
GetBoolData
()
.
Data
=
append
(
dstScalar
.
GetBoolData
()
.
Data
,
srcScalar
.
BoolData
.
Data
...
)
case
*
schemapb
.
ScalarField_IntData
:
dstScalar
.
GetIntData
()
.
Data
=
append
(
dstScalar
.
GetIntData
()
.
Data
,
srcScalar
.
IntData
.
Data
...
)
case
*
schemapb
.
ScalarField_LongData
:
dstScalar
.
GetLongData
()
.
Data
=
append
(
dstScalar
.
GetLongData
()
.
Data
,
srcScalar
.
LongData
.
Data
...
)
case
*
schemapb
.
ScalarField_FloatData
:
dstScalar
.
GetFloatData
()
.
Data
=
append
(
dstScalar
.
GetFloatData
()
.
Data
,
srcScalar
.
FloatData
.
Data
...
)
case
*
schemapb
.
ScalarField_DoubleData
:
dstScalar
.
GetDoubleData
()
.
Data
=
append
(
dstScalar
.
GetDoubleData
()
.
Data
,
srcScalar
.
DoubleData
.
Data
...
)
default
:
log
.
Debug
(
"Query received not supported data type"
,
zap
.
String
(
"field type"
,
fieldData
.
Type
.
String
()))
}
case
*
schemapb
.
FieldData_Vectors
:
dstVector
:=
dst
.
FieldsData
[
i
]
.
GetVectors
()
switch
srcVector
:=
fieldType
.
Vectors
.
Data
.
(
type
)
{
case
*
schemapb
.
VectorField_BinaryVector
:
dstVector
.
Data
.
(
*
schemapb
.
VectorField_BinaryVector
)
.
BinaryVector
=
append
(
dstVector
.
Data
.
(
*
schemapb
.
VectorField_BinaryVector
)
.
BinaryVector
,
srcVector
.
BinaryVector
...
)
case
*
schemapb
.
VectorField_FloatVector
:
dstVector
.
GetFloatVector
()
.
Data
=
append
(
dstVector
.
GetFloatVector
()
.
Data
,
srcVector
.
FloatVector
.
Data
...
)
default
:
log
.
Debug
(
"Query received not supported data type"
,
zap
.
String
(
"field type"
,
fieldData
.
Type
.
String
()))
}
}
}
}
}
func
(
qt
*
queryTask
)
PostExecute
(
ctx
context
.
Context
)
error
{
tr
:=
timerecord
.
NewTimeRecorder
(
"queryTask PostExecute"
)
defer
func
()
{
...
...
@@ -2430,17 +2464,17 @@ func (qt *queryTask) PostExecute(ctx context.Context) error {
log
.
Debug
(
"proxy"
,
zap
.
Int64
(
"Query: wait to finish failed, timeout!, taskID:"
,
qt
.
ID
()))
return
fmt
.
Errorf
(
"queryTask:wait to finish failed, timeout : %d"
,
qt
.
ID
())
case
retrieveResults
:=
<-
qt
.
resultBuf
:
retrieveResult
:=
make
([]
*
internalpb
.
RetrieveResults
,
0
)
filterRetrieveResults
:=
make
([]
*
internalpb
.
RetrieveResults
,
0
)
var
reason
string
for
_
,
partialRetrieveResult
:=
range
retrieveResults
{
if
partialRetrieveResult
.
Status
.
ErrorCode
==
commonpb
.
ErrorCode_Success
{
retrieveResult
=
append
(
retrieveResult
,
partialRetrieveResult
)
filterRetrieveResults
=
append
(
filterRetrieveResults
,
partialRetrieveResult
)
}
else
{
reason
+=
partialRetrieveResult
.
Status
.
Reason
+
"
\n
"
}
}
if
len
(
retrieveResult
)
==
0
{
if
len
(
filterRetrieveResults
)
==
0
{
qt
.
result
=
&
milvuspb
.
QueryResults
{
Status
:
&
commonpb
.
Status
{
ErrorCode
:
commonpb
.
ErrorCode_UnexpectedError
,
...
...
@@ -2452,69 +2486,26 @@ func (qt *queryTask) PostExecute(ctx context.Context) error {
return
errors
.
New
(
reason
)
}
availableQueryNodeNum
:=
0
qt
.
result
=
&
milvuspb
.
QueryResults
{
Status
:
&
commonpb
.
Status
{
ErrorCode
:
commonpb
.
ErrorCode_Success
,
},
FieldsData
:
make
([]
*
schemapb
.
FieldData
,
0
),
}
for
_
,
partialRetrieveResult
:=
range
retrieveResult
{
availableQueryNodeNum
++
if
partialRetrieveResult
.
Ids
==
nil
{
reason
+=
"ids is nil
\n
"
continue
}
else
{
// handles initialization, cannot use idx==0 since first result may be empty
if
len
(
qt
.
result
.
FieldsData
)
==
0
{
qt
.
result
.
FieldsData
=
append
(
qt
.
result
.
FieldsData
,
partialRetrieveResult
.
FieldsData
...
)
}
else
{
for
k
,
fieldData
:=
range
partialRetrieveResult
.
FieldsData
{
switch
fieldType
:=
fieldData
.
Field
.
(
type
)
{
case
*
schemapb
.
FieldData_Scalars
:
switch
scalarType
:=
fieldType
.
Scalars
.
Data
.
(
type
)
{
case
*
schemapb
.
ScalarField_BoolData
:
qt
.
result
.
FieldsData
[
k
]
.
GetScalars
()
.
GetBoolData
()
.
Data
=
append
(
qt
.
result
.
FieldsData
[
k
]
.
GetScalars
()
.
GetBoolData
()
.
Data
,
scalarType
.
BoolData
.
Data
...
)
case
*
schemapb
.
ScalarField_IntData
:
qt
.
result
.
FieldsData
[
k
]
.
GetScalars
()
.
GetIntData
()
.
Data
=
append
(
qt
.
result
.
FieldsData
[
k
]
.
GetScalars
()
.
GetIntData
()
.
Data
,
scalarType
.
IntData
.
Data
...
)
case
*
schemapb
.
ScalarField_LongData
:
qt
.
result
.
FieldsData
[
k
]
.
GetScalars
()
.
GetLongData
()
.
Data
=
append
(
qt
.
result
.
FieldsData
[
k
]
.
GetScalars
()
.
GetLongData
()
.
Data
,
scalarType
.
LongData
.
Data
...
)
case
*
schemapb
.
ScalarField_FloatData
:
qt
.
result
.
FieldsData
[
k
]
.
GetScalars
()
.
GetFloatData
()
.
Data
=
append
(
qt
.
result
.
FieldsData
[
k
]
.
GetScalars
()
.
GetFloatData
()
.
Data
,
scalarType
.
FloatData
.
Data
...
)
case
*
schemapb
.
ScalarField_DoubleData
:
qt
.
result
.
FieldsData
[
k
]
.
GetScalars
()
.
GetDoubleData
()
.
Data
=
append
(
qt
.
result
.
FieldsData
[
k
]
.
GetScalars
()
.
GetDoubleData
()
.
Data
,
scalarType
.
DoubleData
.
Data
...
)
default
:
log
.
Debug
(
"Query received not supported data type"
)
}
case
*
schemapb
.
FieldData_Vectors
:
switch
vectorType
:=
fieldType
.
Vectors
.
Data
.
(
type
)
{
case
*
schemapb
.
VectorField_BinaryVector
:
qt
.
result
.
FieldsData
[
k
]
.
GetVectors
()
.
Data
.
(
*
schemapb
.
VectorField_BinaryVector
)
.
BinaryVector
=
append
(
qt
.
result
.
FieldsData
[
k
]
.
GetVectors
()
.
Data
.
(
*
schemapb
.
VectorField_BinaryVector
)
.
BinaryVector
,
vectorType
.
BinaryVector
...
)
case
*
schemapb
.
VectorField_FloatVector
:
qt
.
result
.
FieldsData
[
k
]
.
GetVectors
()
.
GetFloatVector
()
.
Data
=
append
(
qt
.
result
.
FieldsData
[
k
]
.
GetVectors
()
.
GetFloatVector
()
.
Data
,
vectorType
.
FloatVector
.
Data
...
)
}
default
:
}
}
}
validRetrieveResults
:=
make
([]
*
internalpb
.
RetrieveResults
,
0
)
for
_
,
partialRetrieveResult
:=
range
filterRetrieveResults
{
if
partialRetrieveResult
.
Ids
!=
nil
{
validRetrieveResults
=
append
(
validRetrieveResults
,
partialRetrieveResult
)
}
}
if
availableQueryNodeNum
==
0
{
log
.
Info
(
"Not any valid result found."
,
zap
.
Any
(
"requestID"
,
qt
.
Base
.
MsgID
),
zap
.
Any
(
"requestType"
,
"query"
))
qt
.
result
=
&
milvuspb
.
QueryResults
{
Status
:
&
commonpb
.
Status
{
ErrorCode
:
commonpb
.
ErrorCode_UnexpectedError
,
Reason
:
reason
,
},
}
return
nil
for
_
,
partialRetrieveResult
:=
range
validRetrieveResults
{
copyQueryResultData
(
qt
.
result
,
partialRetrieveResult
)
}
if
len
(
qt
.
result
.
FieldsData
)
==
0
{
log
.
Info
(
"Query result is nil."
,
zap
.
Any
(
"requestID"
,
qt
.
Base
.
MsgID
),
zap
.
Any
(
"requestType"
,
"query"
))
log
.
Info
(
"Query result is nil"
,
zap
.
Any
(
"requestID"
,
qt
.
Base
.
MsgID
),
zap
.
Any
(
"requestType"
,
"query"
))
qt
.
result
=
&
milvuspb
.
QueryResults
{
Status
:
&
commonpb
.
Status
{
ErrorCode
:
commonpb
.
ErrorCode_EmptyCollection
,
...
...
@@ -2539,8 +2530,7 @@ func (qt *queryTask) PostExecute(ctx context.Context) error {
}
}
log
.
Info
(
"Query PostExecute done."
,
zap
.
Any
(
"requestID"
,
qt
.
Base
.
MsgID
),
zap
.
Any
(
"requestType"
,
"query"
))
log
.
Info
(
"Query PostExecute done"
,
zap
.
Any
(
"requestID"
,
qt
.
Base
.
MsgID
),
zap
.
Any
(
"requestType"
,
"query"
))
return
nil
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录