Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
7fc7623b
M
milvus
项目概览
milvus
/
milvus
11 个月 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
7fc7623b
编写于
8月 11, 2021
作者:
G
godchen
提交者:
GitHub
8月 11, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Add vector data memroy existence check (#7044)
Signed-off-by:
N
godchen
<
qingxiang.chen@zilliz.com
>
上级
71d7f407
变更
1
显示空白变更内容
内联
并排
Showing
1 changed file
with
28 addition
and
14 deletion
+28
-14
internal/querynode/query_collection.go
internal/querynode/query_collection.go
+28
-14
未找到文件。
internal/querynode/query_collection.go
浏览文件 @
7fc7623b
...
...
@@ -1069,6 +1069,12 @@ func (q *queryCollection) search(msg queryMsg) error {
}
func
(
q
*
queryCollection
)
fillVectorFieldsData
(
segment
*
Segment
,
result
*
segcorepb
.
RetrieveResults
)
error
{
// If segment is growing, vector data is in memory
if
segment
.
segmentType
==
segmentTypeGrowing
{
log
.
Debug
(
"Segment is growing, all vector data is in memory"
)
log
.
Debug
(
"FillVectorFieldData"
,
zap
.
Any
(
"segmentType"
,
segment
.
segmentType
))
return
nil
}
collection
,
_
:=
q
.
streaming
.
replica
.
getCollectionByID
(
q
.
collectionID
)
schema
:=
&
etcdpb
.
CollectionMeta
{
ID
:
q
.
collectionID
,
...
...
@@ -1078,11 +1084,30 @@ func (q *queryCollection) fillVectorFieldsData(segment *Segment, result *segcore
return
err
}
for
_
,
resultFieldData
:=
range
result
.
FieldsData
{
log
.
Debug
(
"FillVectorFieldData for fieldID"
,
zap
.
Any
(
"fieldID"
,
resultFieldData
.
FieldId
))
// If the vector field doesn't have index. Vector data is in memory for
// brute force search. No need to download data from remote.
_
,
ok
:=
segment
.
indexInfos
[
resultFieldData
.
FieldId
]
if
!
ok
{
log
.
Debug
(
"FillVectorFieldData fielD doesn't have index"
,
zap
.
Any
(
"fielD"
,
resultFieldData
.
FieldId
))
continue
}
vecFieldInfo
,
err
:=
segment
.
getVectorFieldInfo
(
resultFieldData
.
FieldId
)
if
err
!=
nil
{
continue
}
log
.
Debug
(
"FillVectorFieldData"
,
zap
.
Any
(
"fieldID"
,
resultFieldData
.
FieldId
))
dim
:=
resultFieldData
.
GetVectors
()
.
GetDim
()
log
.
Debug
(
"FillVectorFieldData"
,
zap
.
Any
(
"dim"
,
dim
))
fieldSchema
,
err
:=
schemaHelper
.
GetFieldFromID
(
resultFieldData
.
FieldId
)
if
err
!=
nil
{
return
err
}
dataType
:=
fieldSchema
.
DataType
log
.
Debug
(
"FillVectorFieldData"
,
zap
.
Any
(
"datatype"
,
dataType
))
data
:=
resultFieldData
.
GetVectors
()
.
GetData
()
for
i
,
offset
:=
range
result
.
Offset
{
var
vecPath
string
...
...
@@ -1100,19 +1125,9 @@ func (q *queryCollection) fillVectorFieldsData(segment *Segment, result *segcore
return
err
}
dim
:=
resultFieldData
.
GetVectors
()
.
GetDim
()
log
.
Debug
(
"FillVectorFieldData"
,
zap
.
Any
(
"dim"
,
dim
))
schema
,
err
:=
schemaHelper
.
GetFieldFromID
(
resultFieldData
.
FieldId
)
if
err
!=
nil
{
return
err
}
dataType
:=
schema
.
DataType
log
.
Debug
(
"FillVectorFieldData"
,
zap
.
Any
(
"datatype"
,
dataType
))
switch
dataType
{
case
schemapb
.
DataType_BinaryVector
:
rowBytes
:=
dim
/
8
x
:=
resultFieldData
.
GetVectors
()
.
GetData
()
.
(
*
schemapb
.
VectorField_BinaryVector
)
content
:=
make
([]
byte
,
rowBytes
)
_
,
err
:=
q
.
vcm
.
ReadAt
(
vecPath
,
content
,
offset
*
rowBytes
)
if
err
!=
nil
{
...
...
@@ -1121,9 +1136,8 @@ func (q *queryCollection) fillVectorFieldsData(segment *Segment, result *segcore
log
.
Debug
(
"FillVectorFieldData"
,
zap
.
Any
(
"binaryVectorResult"
,
content
))
resultLen
:=
dim
/
8
copy
(
x
.
BinaryVector
[
i
*
int
(
resultLen
)
:
(
i
+
1
)
*
int
(
resultLen
)],
content
)
copy
(
data
.
(
*
schemapb
.
VectorField_BinaryVector
)
.
BinaryVector
[
i
*
int
(
resultLen
)
:
(
i
+
1
)
*
int
(
resultLen
)],
content
)
case
schemapb
.
DataType_FloatVector
:
x
:=
resultFieldData
.
GetVectors
()
.
GetData
()
.
(
*
schemapb
.
VectorField_FloatVector
)
rowBytes
:=
dim
*
4
content
:=
make
([]
byte
,
rowBytes
)
_
,
err
:=
q
.
vcm
.
ReadAt
(
vecPath
,
content
,
offset
*
rowBytes
)
...
...
@@ -1139,7 +1153,7 @@ func (q *queryCollection) fillVectorFieldsData(segment *Segment, result *segcore
log
.
Debug
(
"FillVectorFieldData"
,
zap
.
Any
(
"floatVectorResult"
,
floatResult
))
resultLen
:=
dim
copy
(
x
.
FloatVector
.
Data
[
i
*
int
(
resultLen
)
:
(
i
+
1
)
*
int
(
resultLen
)],
floatResult
)
copy
(
data
.
(
*
schemapb
.
VectorField_FloatVector
)
.
FloatVector
.
Data
[
i
*
int
(
resultLen
)
:
(
i
+
1
)
*
int
(
resultLen
)],
floatResult
)
}
}
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录