Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
cc9ac00e
M
milvus
项目概览
milvus
/
milvus
9 个月 前同步成功
通知
260
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
前往新版Gitcode,体验更适合开发者的 AI 搜索 >>
提交
cc9ac00e
编写于
9月 04, 2020
作者:
W
Wang Xiangyu
浏览文件
操作
浏览文件
下载
差异文件
solve conflicts
Signed-off-by:
N
Wang Xiangyu
<
xy.wang@zilliz.com
>
上级
7e863578
961d37c9
变更
10
隐藏空白更改
内联
并排
Showing
10 changed file
with
118 addition
and
1 deletion
+118
-1
CHANGELOG.md
CHANGELOG.md
+2
-0
core/src/query/GeneralQuery.h
core/src/query/GeneralQuery.h
+1
-0
core/src/scheduler/selector/FaissFlatPass.cpp
core/src/scheduler/selector/FaissFlatPass.cpp
+3
-0
core/src/scheduler/selector/FaissIVFFlatPass.cpp
core/src/scheduler/selector/FaissIVFFlatPass.cpp
+10
-1
core/src/scheduler/selector/FaissIVFPQPass.cpp
core/src/scheduler/selector/FaissIVFPQPass.cpp
+9
-0
core/src/scheduler/selector/FaissIVFSQ8HPass.cpp
core/src/scheduler/selector/FaissIVFSQ8HPass.cpp
+9
-0
core/src/scheduler/selector/FaissIVFSQ8Pass.cpp
core/src/scheduler/selector/FaissIVFSQ8Pass.cpp
+9
-0
core/src/scheduler/task/SearchTask.cpp
core/src/scheduler/task/SearchTask.cpp
+62
-0
core/src/scheduler/task/SearchTask.h
core/src/scheduler/task/SearchTask.h
+7
-0
core/src/server/web_impl/handler/WebRequestHandler.cpp
core/src/server/web_impl/handler/WebRequestHandler.cpp
+6
-0
未找到文件。
CHANGELOG.md
浏览文件 @
cc9ac00e
...
...
@@ -36,6 +36,8 @@ Please mark all changes in change log and use the issue from GitHub
-
\#
3453 Fix server crashed during stability test
-
\#
3482 Server crashed during adding entities
-
\#
3490 Fix ut DBTest.DeleteEntitiesTest assert fail
-
\#
3511 SearchTask::nq() should not return 0
-
\#
3514 Search failed with regex tag name
## Feature
-
\#
2319 Redo metadata to support MVCC
...
...
core/src/query/GeneralQuery.h
浏览文件 @
cc9ac00e
...
...
@@ -120,6 +120,7 @@ struct Query {
std
::
vector
<
std
::
string
>
field_names
;
std
::
set
<
std
::
string
>
index_fields
;
std
::
unordered_map
<
std
::
string
,
std
::
string
>
metric_types
;
std
::
string
index_type
;
};
using
QueryPtr
=
std
::
shared_ptr
<
Query
>
;
...
...
core/src/scheduler/selector/FaissFlatPass.cpp
浏览文件 @
cc9ac00e
...
...
@@ -44,6 +44,9 @@ FaissFlatPass::Run(const TaskPtr& task) {
}
auto
search_task
=
std
::
static_pointer_cast
<
SearchTask
>
(
task
);
if
(
search_task
->
IndexType
()
!=
knowhere
::
IndexEnum
::
INDEX_FAISS_IDMAP
)
{
return
false
;
}
ResourcePtr
res_ptr
;
if
(
!
gpu_enable_
)
{
...
...
core/src/scheduler/selector/FaissIVFFlatPass.cpp
浏览文件 @
cc9ac00e
...
...
@@ -12,6 +12,8 @@
#include "scheduler/selector/FaissIVFFlatPass.h"
#include "cache/GpuCacheMgr.h"
#include "config/ServerConfig.h"
#include "faiss/gpu/utils/DeviceUtils.h"
#include "knowhere/index/vector_index/helpers/IndexParameter.h"
#include "scheduler/SchedInst.h"
#include "scheduler/Utils.h"
#include "scheduler/task/SearchTask.h"
...
...
@@ -45,13 +47,20 @@ FaissIVFFlatPass::Run(const TaskPtr& task) {
}
auto
search_task
=
std
::
static_pointer_cast
<
SearchTask
>
(
task
);
if
(
search_task
->
IndexType
()
!=
knowhere
::
IndexEnum
::
INDEX_FAISS_IVFFLAT
)
{
return
false
;
}
ResourcePtr
res_ptr
;
if
(
!
gpu_enable_
)
{
LOG_SERVER_DEBUG_
<<
LogOut
(
"FaissIVFFlatPass: gpu disable, specify cpu to search!"
);
res_ptr
=
ResMgrInst
::
GetInstance
()
->
GetResource
(
"cpu"
);
}
else
if
(
search_task
->
nq
()
<
threshold_
)
{
LOG_SERVER_DEBUG_
<<
LogOut
(
"FaissIVFFlatPass: nq < gpu_search_threshold, specify cpu to search!"
);
LOG_SERVER_DEBUG_
<<
LogOut
(
"FaissIVFFlatPass: nq < gpu_search_threshold, specify cpu to search! "
);
res_ptr
=
ResMgrInst
::
GetInstance
()
->
GetResource
(
"cpu"
);
}
else
if
(
search_task
->
ExtraParam
()[
knowhere
::
IndexParams
::
nprobe
].
get
<
int64_t
>
()
>
faiss
::
gpu
::
getMaxKSelection
())
{
LOG_SERVER_DEBUG_
<<
LogOut
(
"FaissIVFFlatPass: nprobe > gpu_max_nprobe_threshold, specify cpu to search!"
);
res_ptr
=
ResMgrInst
::
GetInstance
()
->
GetResource
(
"cpu"
);
}
else
{
LOG_SERVER_DEBUG_
<<
LogOut
(
"FaissIVFFlatPass: nq >= gpu_search_threshold, specify gpu %d to search!"
,
...
...
core/src/scheduler/selector/FaissIVFPQPass.cpp
浏览文件 @
cc9ac00e
...
...
@@ -12,6 +12,8 @@
#include "scheduler/selector/FaissIVFPQPass.h"
#include "cache/GpuCacheMgr.h"
#include "config/ServerConfig.h"
#include "faiss/gpu/utils/DeviceUtils.h"
#include "knowhere/index/vector_index/helpers/IndexParameter.h"
#include "scheduler/SchedInst.h"
#include "scheduler/Utils.h"
#include "scheduler/task/SearchTask.h"
...
...
@@ -47,6 +49,9 @@ FaissIVFPQPass::Run(const TaskPtr& task) {
}
auto
search_task
=
std
::
static_pointer_cast
<
SearchTask
>
(
task
);
if
(
search_task
->
IndexType
()
!=
knowhere
::
IndexEnum
::
INDEX_FAISS_IVFPQ
)
{
return
false
;
}
ResourcePtr
res_ptr
;
if
(
!
gpu_enable_
)
{
...
...
@@ -55,6 +60,10 @@ FaissIVFPQPass::Run(const TaskPtr& task) {
}
else
if
(
search_task
->
nq
()
<
threshold_
)
{
LOG_SERVER_DEBUG_
<<
LogOut
(
"FaissIVFPQPass: nq < gpu_search_threshold, specify cpu to search!"
);
res_ptr
=
ResMgrInst
::
GetInstance
()
->
GetResource
(
"cpu"
);
}
else
if
(
search_task
->
ExtraParam
()[
knowhere
::
IndexParams
::
nprobe
].
get
<
int64_t
>
()
>
faiss
::
gpu
::
getMaxKSelection
())
{
LOG_SERVER_DEBUG_
<<
LogOut
(
"FaissIVFFlatPass: nprobe > gpu_max_nprobe_threshold, specify cpu to search!"
);
res_ptr
=
ResMgrInst
::
GetInstance
()
->
GetResource
(
"cpu"
);
}
else
{
LOG_SERVER_DEBUG_
<<
LogOut
(
"FaissIVFPQPass: nq >= gpu_search_threshold, specify gpu %d to search!"
,
search_gpus_
[
idx_
]);
...
...
core/src/scheduler/selector/FaissIVFSQ8HPass.cpp
浏览文件 @
cc9ac00e
...
...
@@ -13,6 +13,8 @@
#include "scheduler/selector/FaissIVFSQ8HPass.h"
#include "cache/GpuCacheMgr.h"
#include "config/ServerConfig.h"
#include "faiss/gpu/utils/DeviceUtils.h"
#include "knowhere/index/vector_index/helpers/IndexParameter.h"
#include "scheduler/SchedInst.h"
#include "scheduler/Utils.h"
#include "scheduler/task/SearchTask.h"
...
...
@@ -44,6 +46,9 @@ FaissIVFSQ8HPass::Run(const TaskPtr& task) {
}
auto
search_task
=
std
::
static_pointer_cast
<
SearchTask
>
(
task
);
if
(
search_task
->
IndexType
()
!=
knowhere
::
IndexEnum
::
INDEX_FAISS_IVFSQ8H
)
{
return
false
;
}
ResourcePtr
res_ptr
;
if
(
!
gpu_enable_
)
{
...
...
@@ -53,6 +58,10 @@ FaissIVFSQ8HPass::Run(const TaskPtr& task) {
if
(
search_task
->
nq
()
<
threshold_
)
{
LOG_SERVER_DEBUG_
<<
LogOut
(
"FaissIVFSQ8HPass: nq < gpu_search_threshold, specify cpu to search!"
);
res_ptr
=
ResMgrInst
::
GetInstance
()
->
GetResource
(
"cpu"
);
}
else
if
(
search_task
->
ExtraParam
()[
knowhere
::
IndexParams
::
nprobe
].
get
<
int64_t
>
()
>
faiss
::
gpu
::
getMaxKSelection
())
{
LOG_SERVER_DEBUG_
<<
LogOut
(
"FaissIVFFlatPass: nprobe > gpu_max_nprobe_threshold, specify cpu to search!"
);
res_ptr
=
ResMgrInst
::
GetInstance
()
->
GetResource
(
"cpu"
);
}
else
{
LOG_SERVER_DEBUG_
<<
LogOut
(
"FaissIVFSQ8HPass: nq >= gpu_search_threshold, specify gpu %d to search!"
,
search_gpus_
[
idx_
]);
...
...
core/src/scheduler/selector/FaissIVFSQ8Pass.cpp
浏览文件 @
cc9ac00e
...
...
@@ -12,6 +12,8 @@
#include "scheduler/selector/FaissIVFSQ8Pass.h"
#include "cache/GpuCacheMgr.h"
#include "config/ServerConfig.h"
#include "faiss/gpu/utils/DeviceUtils.h"
#include "knowhere/index/vector_index/helpers/IndexParameter.h"
#include "scheduler/SchedInst.h"
#include "scheduler/Utils.h"
#include "scheduler/task/SearchTask.h"
...
...
@@ -45,6 +47,9 @@ FaissIVFSQ8Pass::Run(const TaskPtr& task) {
}
auto
search_task
=
std
::
static_pointer_cast
<
SearchTask
>
(
task
);
if
(
search_task
->
IndexType
()
!=
knowhere
::
IndexEnum
::
INDEX_FAISS_IVFSQ8
)
{
return
false
;
}
ResourcePtr
res_ptr
;
if
(
!
gpu_enable_
)
{
...
...
@@ -53,6 +58,10 @@ FaissIVFSQ8Pass::Run(const TaskPtr& task) {
}
else
if
(
search_task
->
nq
()
<
threshold_
)
{
LOG_SERVER_DEBUG_
<<
LogOut
(
"FaissIVFSQ8Pass: nq < gpu_search_threshold, specify cpu to search!"
);
res_ptr
=
ResMgrInst
::
GetInstance
()
->
GetResource
(
"cpu"
);
}
else
if
(
search_task
->
ExtraParam
()[
knowhere
::
IndexParams
::
nprobe
].
get
<
int64_t
>
()
>
faiss
::
gpu
::
getMaxKSelection
())
{
LOG_SERVER_DEBUG_
<<
LogOut
(
"FaissIVFFlatPass: nprobe > gpu_max_nprobe_threshold, specify cpu to search!"
);
res_ptr
=
ResMgrInst
::
GetInstance
()
->
GetResource
(
"cpu"
);
}
else
{
LOG_SERVER_DEBUG_
<<
LogOut
(
"FaissIVFSQ8Pass: nq >= gpu_search_threshold, specify gpu %d to search!"
,
search_gpus_
[
idx_
]);
...
...
core/src/scheduler/task/SearchTask.cpp
浏览文件 @
cc9ac00e
...
...
@@ -13,6 +13,7 @@
#include <fiu/fiu-local.h>
#include <src/index/thirdparty/faiss/IndexFlat.h>
#include <algorithm>
#include <memory>
#include <string>
...
...
@@ -220,8 +221,69 @@ SearchTask::MergeTopkToResultSet(const engine::ResultIds& src_ids, const engine:
int64_t
SearchTask
::
nq
()
{
if
(
query_ptr_
)
{
auto
vector_query
=
query_ptr_
->
vectors
.
begin
();
if
(
vector_query
!=
query_ptr_
->
vectors
.
end
())
{
if
(
vector_query
->
second
)
{
auto
vector_param
=
vector_query
->
second
;
auto
field_visitor
=
snapshot_
->
GetField
(
vector_query
->
second
->
field_name
);
if
(
field_visitor
)
{
if
(
field_visitor
->
GetParams
().
contains
(
engine
::
PARAM_DIMENSION
))
{
int64_t
dim
=
field_visitor
->
GetParams
()[
engine
::
PARAM_DIMENSION
];
if
(
!
vector_param
->
query_vector
.
float_data
.
empty
())
{
return
vector_param
->
query_vector
.
float_data
.
size
()
/
dim
;
}
else
if
(
!
vector_param
->
query_vector
.
binary_data
.
empty
())
{
return
vector_param
->
query_vector
.
binary_data
.
size
()
*
8
/
dim
;
}
}
}
}
}
}
return
0
;
}
milvus
::
json
SearchTask
::
ExtraParam
()
{
milvus
::
json
param
;
if
(
query_ptr_
)
{
auto
vector_query
=
query_ptr_
->
vectors
.
begin
();
if
(
vector_query
!=
query_ptr_
->
vectors
.
end
())
{
if
(
vector_query
->
second
)
{
return
vector_query
->
second
->
extra_params
;
}
}
}
return
param
;
}
std
::
string
SearchTask
::
IndexType
()
{
if
(
!
index_type_
.
empty
())
{
return
index_type_
;
}
auto
seg_visitor
=
engine
::
SegmentVisitor
::
Build
(
snapshot_
,
segment_id_
);
index_type_
=
"FLAT"
;
if
(
seg_visitor
)
{
for
(
const
auto
&
name
:
query_ptr_
->
index_fields
)
{
auto
field_visitor
=
seg_visitor
->
GetFieldVisitor
(
name
);
auto
type
=
field_visitor
->
GetField
()
->
GetFtype
();
if
(
!
field_visitor
)
{
continue
;
}
if
(
type
==
engine
::
DataType
::
VECTOR_FLOAT
||
type
==
engine
::
DataType
::
VECTOR_BINARY
)
{
auto
fe_visitor
=
field_visitor
->
GetElementVisitor
(
engine
::
FieldElementType
::
FET_INDEX
);
if
(
fe_visitor
)
{
auto
element
=
fe_visitor
->
GetElement
();
index_type_
=
element
->
GetTypeName
();
}
return
index_type_
;
}
}
}
return
index_type_
;
}
}
// namespace scheduler
}
// namespace milvus
core/src/scheduler/task/SearchTask.h
浏览文件 @
cc9ac00e
...
...
@@ -52,6 +52,12 @@ class SearchTask : public Task {
int64_t
nq
();
milvus
::
json
ExtraParam
();
std
::
string
IndexType
();
private:
void
CreateExecEngine
();
...
...
@@ -63,6 +69,7 @@ class SearchTask : public Task {
const
engine
::
DBOptions
&
options_
;
query
::
QueryPtr
query_ptr_
;
engine
::
snapshot
::
ID_TYPE
segment_id_
;
std
::
string
index_type_
;
engine
::
ExecutionEnginePtr
execution_engine_
;
...
...
core/src/server/web_impl/handler/WebRequestHandler.cpp
浏览文件 @
cc9ac00e
...
...
@@ -286,6 +286,12 @@ WebRequestHandler::GetCollectionMetaInfo(const std::string& collection_name, nlo
field_json
[
"extra_params"
]
=
field
.
second
.
field_params_
;
json_out
[
"fields"
].
push_back
(
field_json
);
}
if
(
schema
.
extra_params_
.
contains
(
engine
::
PARAM_SEGMENT_ROW_COUNT
))
{
json_out
[
engine
::
PARAM_SEGMENT_ROW_COUNT
]
=
schema
.
extra_params_
[
engine
::
PARAM_SEGMENT_ROW_COUNT
];
}
if
(
schema
.
extra_params_
.
contains
(
engine
::
PARAM_UID_AUTOGEN
))
{
json_out
[
engine
::
PARAM_UID_AUTOGEN
]
=
schema
.
extra_params_
[
engine
::
PARAM_UID_AUTOGEN
];
}
return
Status
::
OK
();
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录