Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
315aa0af
M
milvus
项目概览
milvus
/
milvus
12 个月 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
315aa0af
编写于
11月 08, 2021
作者:
C
Cai Yudong
提交者:
GitHub
11月 08, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Merge GetResultData and ResetResultData into ReduceResultData (#11396)
Signed-off-by:
N
yudong.cai
<
yudong.cai@zilliz.com
>
上级
db6ecd58
变更
1
显示空白变更内容
内联
并排
Showing
1 changed file
with
20 addition
and
37 deletion
+20
-37
internal/core/src/segcore/reduce_c.cpp
internal/core/src/segcore/reduce_c.cpp
+20
-37
未找到文件。
internal/core/src/segcore/reduce_c.cpp
浏览文件 @
315aa0af
...
...
@@ -67,28 +67,31 @@ DeleteMarshaledHits(CMarshaledHits c_marshaled_hits) {
//}
void
GetResultData
(
std
::
vector
<
std
::
vector
<
int64_t
>>&
search_records
,
std
::
vector
<
SearchResult
*>&
search_results
,
int64_t
nq
,
int64_t
topk
)
{
ReduceResultData
(
std
::
vector
<
SearchResult
*>&
search_results
,
int64_t
nq
,
int64_t
topk
)
{
AssertInfo
(
topk
>
0
,
"topk must greater than 0"
);
auto
num_segments
=
search_results
.
size
();
AssertInfo
(
num_segments
>
0
,
"num segment must greater than 0"
);
for
(
int
i
=
0
;
i
<
num_segments
;
i
++
)
{
auto
search_result
=
search_results
[
i
];
AssertInfo
(
search_result
!=
nullptr
,
"search result must not equal to nullptr"
);
AssertInfo
(
search_result
->
primary_keys_
.
size
()
==
nq
*
topk
,
"incorrect search result primary key size"
);
AssertInfo
(
search_result
->
result_distances_
.
size
()
==
nq
*
topk
,
"incorrect search result distance size"
);
}
std
::
vector
<
std
::
vector
<
int64_t
>>
search_records
(
num_segments
);
std
::
unordered_set
<
int64_t
>
pk_set
;
int64_t
skip_dup_cnt
=
0
;
// reduce search results
for
(
int64_t
qi
=
0
;
qi
<
nq
;
qi
++
)
{
std
::
vector
<
SearchResultPair
>
result_pairs
;
int64_t
base_offset
=
qi
*
topk
;
for
(
int
j
=
0
;
j
<
num_segments
;
++
j
)
{
auto
search_result
=
search_results
[
j
];
AssertInfo
(
search_result
!=
nullptr
,
"search result must not equal to nullptr"
);
AssertInfo
(
search_result
->
primary_keys_
.
size
()
==
nq
*
topk
,
"incorrect search result primary key size"
);
AssertInfo
(
search_result
->
result_distances_
.
size
()
==
nq
*
topk
,
"incorrect search result distance size"
);
for
(
int
i
=
0
;
i
<
num_segments
;
i
++
)
{
auto
search_result
=
search_results
[
i
];
auto
primary_key
=
search_result
->
primary_keys_
[
base_offset
];
auto
distance
=
search_result
->
result_distances_
[
base_offset
];
result_pairs
.
push_back
(
SearchResultPair
(
primary_key
,
distance
,
search_result
,
j
,
base_offset
,
base_offset
+
topk
));
SearchResultPair
(
primary_key
,
distance
,
search_result
,
i
,
base_offset
,
base_offset
+
topk
));
}
int64_t
curr_offset
=
base_offset
;
...
...
@@ -125,18 +128,11 @@ GetResultData(std::vector<std::vector<int64_t>>& search_records,
}
#endif
}
if
(
skip_dup_cnt
>
0
)
{
LOG_SEGCORE_DEBUG_
<<
"skip duplicated search result, count = "
<<
skip_dup_cnt
;
}
}
void
ResetSearchResult
(
std
::
vector
<
std
::
vector
<
int64_t
>>&
search_records
,
std
::
vector
<
SearchResult
*>&
search_results
)
{
auto
num_segments
=
search_results
.
size
();
AssertInfo
(
num_segments
>
0
,
"num segment must greater than 0"
);
// after reduce, remove redundant values in primary_keys, result_distances and internal_seg_offsets
for
(
int
i
=
0
;
i
<
num_segments
;
i
++
)
{
auto
search_result
=
search_results
[
i
];
AssertInfo
(
search_result
!=
nullptr
,
"search result must not equal to nullptr"
);
if
(
search_result
->
result_offsets_
.
size
()
==
0
)
{
continue
;
}
...
...
@@ -144,23 +140,12 @@ ResetSearchResult(std::vector<std::vector<int64_t>>& search_records, std::vector
std
::
vector
<
int64_t
>
primary_keys
;
std
::
vector
<
float
>
result_distances
;
std
::
vector
<
int64_t
>
internal_seg_offsets
;
int64_t
primary_key
;
float
distance
;
int64_t
internal_seg_offset
;
for
(
int
j
=
0
;
j
<
search_records
[
i
].
size
();
j
++
)
{
auto
&
offset
=
search_records
[
i
][
j
];
if
(
offset
!=
INVALID_OFFSET
)
{
primary_key
=
search_result
->
primary_keys_
[
offset
];
distance
=
search_result
->
result_distances_
[
offset
];
internal_seg_offset
=
search_result
->
internal_seg_offsets_
[
offset
];
}
else
{
primary_key
=
INVALID_ID
;
distance
=
MAXFLOAT
;
internal_seg_offset
=
INVALID_SEG_OFFSET
;
}
primary_keys
.
push_back
(
primary_key
);
result_distances
.
push_back
(
distance
);
internal_seg_offsets
.
push_back
(
internal_seg_offset
);
primary_keys
.
push_back
(
offset
!=
INVALID_OFFSET
?
search_result
->
primary_keys_
[
offset
]
:
INVALID_ID
);
result_distances
.
push_back
(
offset
!=
INVALID_OFFSET
?
search_result
->
result_distances_
[
offset
]
:
MAXFLOAT
);
internal_seg_offsets
.
push_back
(
offset
!=
INVALID_OFFSET
?
search_result
->
internal_seg_offsets_
[
offset
]
:
INVALID_SEG_OFFSET
);
}
search_result
->
primary_keys_
=
primary_keys
;
...
...
@@ -179,7 +164,6 @@ ReduceSearchResultsAndFillData(CSearchPlan c_plan, CSearchResult* c_search_resul
}
auto
topk
=
search_results
[
0
]
->
topk_
;
auto
num_queries
=
search_results
[
0
]
->
num_queries_
;
std
::
vector
<
std
::
vector
<
int64_t
>>
search_records
(
num_segments
);
// get primary keys for duplicates removal
for
(
auto
&
search_result
:
search_results
)
{
...
...
@@ -187,8 +171,7 @@ ReduceSearchResultsAndFillData(CSearchPlan c_plan, CSearchResult* c_search_resul
segment
->
FillPrimaryKeys
(
plan
,
*
search_result
);
}
GetResultData
(
search_records
,
search_results
,
num_queries
,
topk
);
ResetSearchResult
(
search_records
,
search_results
);
ReduceResultData
(
search_results
,
num_queries
,
topk
);
// fill in other entities
for
(
auto
&
search_result
:
search_results
)
{
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录