Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
milvus
提交
9af8a46f
milvus
项目概览
BaiXuePrincess
/
milvus
与 Fork 源项目一致
从无法访问的项目Fork
通知
7
Star
4
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
9af8a46f
编写于
9月 07, 2019
作者:
W
wxyu
浏览文件
操作
浏览文件
下载
差异文件
solve conflicts
Former-commit-id: 291015fff6653bd61c931579e77e78645d2db32a
上级
08750101
52bf617b
变更
7
隐藏空白更改
内联
并排
Showing
7 changed file
with
30 addition
and
26 deletion
+30
-26
cpp/CHANGELOG.md
cpp/CHANGELOG.md
+1
-0
cpp/src/scheduler/task/SearchTask.cpp
cpp/src/scheduler/task/SearchTask.cpp
+11
-9
cpp/src/server/grpc_impl/GrpcRequestTask.cpp
cpp/src/server/grpc_impl/GrpcRequestTask.cpp
+4
-6
cpp/src/utils/ValidationUtil.cpp
cpp/src/utils/ValidationUtil.cpp
+1
-1
cpp/src/wrapper/knowhere/vec_impl.cpp
cpp/src/wrapper/knowhere/vec_impl.cpp
+1
-0
cpp/src/wrapper/knowhere/vec_index.cpp
cpp/src/wrapper/knowhere/vec_index.cpp
+6
-2
cpp/src/wrapper/knowhere/vec_index.h
cpp/src/wrapper/knowhere/vec_index.h
+6
-8
未找到文件。
cpp/CHANGELOG.md
浏览文件 @
9af8a46f
...
...
@@ -92,6 +92,7 @@ Please mark all change in change log and use the ticket from JIRA.
-
MS-487 - Define metric type in CreateTable
-
MS-488 - Improve code format in scheduler
-
MS-495 - cmake: integrated knowhere
-
MS-496 - Change the top_k limitation from 1024 to 2048
-
MS-502 - Update tasktable_test in scheduler
-
MS-504 - Update node_test in scheduler
-
MS-505 - Install core unit test and add to coverage
...
...
cpp/src/scheduler/task/SearchTask.cpp
浏览文件 @
9af8a46f
...
...
@@ -106,6 +106,8 @@ XSearchTask::Load(LoadType type, uint8_t device_id) {
index_engine_
->
CopyToCpu
();
}
else
{
// TODO: exception
std
::
string
msg
=
"Wrong load type"
;
ENGINE_LOG_ERROR
<<
msg
;
}
}
catch
(
std
::
exception
&
ex
)
{
//typical error: out of disk space or permition denied
...
...
@@ -150,17 +152,17 @@ XSearchTask::Execute() {
server
::
CollectDurationMetrics
metrics
(
index_type_
);
std
::
vector
<
long
>
output_ids
;
std
::
vector
<
float
>
output_dist
e
nce
;
std
::
vector
<
float
>
output_dist
a
nce
;
for
(
auto
&
context
:
search_contexts_
)
{
//step 1: allocate memory
auto
inner_k
=
context
->
topk
();
auto
nprobe
=
context
->
nprobe
();
output_ids
.
resize
(
inner_k
*
context
->
nq
());
output_dist
e
nce
.
resize
(
inner_k
*
context
->
nq
());
output_dist
a
nce
.
resize
(
inner_k
*
context
->
nq
());
try
{
//step 2: search
index_engine_
->
Search
(
context
->
nq
(),
context
->
vectors
(),
inner_k
,
nprobe
,
output_dist
e
nce
.
data
(),
index_engine_
->
Search
(
context
->
nq
(),
context
->
vectors
(),
inner_k
,
nprobe
,
output_dist
a
nce
.
data
(),
output_ids
.
data
());
double
span
=
rc
.
RecordSection
(
"do search for context:"
+
context
->
Identity
());
...
...
@@ -170,12 +172,12 @@ XSearchTask::Execute() {
//step 3: cluster result
SearchContext
::
ResultSet
result_set
;
auto
spec_k
=
index_engine_
->
Count
()
<
context
->
topk
()
?
index_engine_
->
Count
()
:
context
->
topk
();
XSearchTask
::
ClusterResult
(
output_ids
,
output_dist
e
nce
,
context
->
nq
(),
spec_k
,
result_set
);
XSearchTask
::
ClusterResult
(
output_ids
,
output_dist
a
nce
,
context
->
nq
(),
spec_k
,
result_set
);
span
=
rc
.
RecordSection
(
"cluster result for context:"
+
context
->
Identity
());
context
->
AccumReduceCost
(
span
);
//step 4: pick up topk result
//
step 4: pick up topk result
XSearchTask
::
TopkResult
(
result_set
,
inner_k
,
metric_l2
,
context
->
GetResult
());
span
=
rc
.
RecordSection
(
"reduce topk for context:"
+
context
->
Identity
());
...
...
@@ -197,13 +199,13 @@ XSearchTask::Execute() {
}
Status
XSearchTask
::
ClusterResult
(
const
std
::
vector
<
long
>
&
output_ids
,
const
std
::
vector
<
float
>
&
output_dist
e
nce
,
const
std
::
vector
<
float
>
&
output_dist
a
nce
,
uint64_t
nq
,
uint64_t
topk
,
SearchContext
::
ResultSet
&
result_set
)
{
if
(
output_ids
.
size
()
<
nq
*
topk
||
output_dist
e
nce
.
size
()
<
nq
*
topk
)
{
if
(
output_ids
.
size
()
<
nq
*
topk
||
output_dist
a
nce
.
size
()
<
nq
*
topk
)
{
std
::
string
msg
=
"Invalid id array size: "
+
std
::
to_string
(
output_ids
.
size
())
+
" distance array size: "
+
std
::
to_string
(
output_dist
e
nce
.
size
());
" distance array size: "
+
std
::
to_string
(
output_dist
a
nce
.
size
());
ENGINE_LOG_ERROR
<<
msg
;
return
Status
(
DB_ERROR
,
msg
);
}
...
...
@@ -220,7 +222,7 @@ Status XSearchTask::ClusterResult(const std::vector<long> &output_ids,
if
(
output_ids
[
index
]
<
0
)
{
continue
;
}
id_distance
.
push_back
(
std
::
make_pair
(
output_ids
[
index
],
output_dist
e
nce
[
index
]));
id_distance
.
push_back
(
std
::
make_pair
(
output_ids
[
index
],
output_dist
a
nce
[
index
]));
}
result_set
[
i
]
=
id_distance
;
}
...
...
cpp/src/server/grpc_impl/GrpcRequestTask.cpp
浏览文件 @
9af8a46f
...
...
@@ -652,7 +652,7 @@ SearchTask::OnExecute() {
search_param_
->
query_record_array
(
i
).
vector_data
().
data
(),
table_info
.
dimension_
*
sizeof
(
float
));
}
rc
.
ElapseFromBegi
n
(
"prepare vector data"
);
rc
.
RecordSectio
n
(
"prepare vector data"
);
//step 6: search vectors
engine
::
QueryResults
results
;
...
...
@@ -666,7 +666,7 @@ SearchTask::OnExecute() {
record_count
,
nprobe
,
vec_f
.
data
(),
dates
,
results
);
}
rc
.
ElapseFromBegi
n
(
"search vectors from engine"
);
rc
.
RecordSectio
n
(
"search vectors from engine"
);
if
(
!
stat
.
ok
())
{
return
SetError
(
DB_META_TRANSACTION_FAILED
,
stat
.
ToString
());
}
...
...
@@ -681,8 +681,6 @@ SearchTask::OnExecute() {
return
SetError
(
SERVER_ILLEGAL_SEARCH_RESULT
,
msg
);
}
rc
.
ElapseFromBegin
(
"do search"
);
//step 7: construct result array
for
(
auto
&
result
:
results
)
{
::
milvus
::
grpc
::
TopKQueryResult
*
topk_query_result
=
topk_result_list
->
add_topk_query_result
();
...
...
@@ -698,7 +696,7 @@ SearchTask::OnExecute() {
#endif
//step 8: print time cost percent
double
span_result
=
rc
.
RecordSection
(
"construct result
"
);
rc
.
RecordSection
(
"construct result and send
"
);
rc
.
ElapseFromBegin
(
"totally cost"
);
...
...
@@ -972,4 +970,4 @@ DropIndexTask::OnExecute() {
}
}
}
}
\ No newline at end of file
}
cpp/src/utils/ValidationUtil.cpp
浏览文件 @
9af8a46f
...
...
@@ -94,7 +94,7 @@ ValidationUtil::ValidateTableIndexMetricType(int32_t metric_type) {
ErrorCode
ValidationUtil
::
ValidateSearchTopk
(
int64_t
top_k
,
const
engine
::
meta
::
TableSchema
&
table_schema
)
{
if
(
top_k
<=
0
||
top_k
>
1024
)
{
if
(
top_k
<=
0
||
top_k
>
2048
)
{
return
SERVER_INVALID_TOPK
;
}
...
...
cpp/src/wrapper/knowhere/vec_impl.cpp
浏览文件 @
9af8a46f
...
...
@@ -103,6 +103,7 @@ ErrorCode VecIndexImpl::Search(const long &nq, const float *xq, float *dist, lon
// TODO(linxj): avoid copy here.
memcpy
(
ids
,
p_ids
,
sizeof
(
int64_t
)
*
nq
*
k
);
memcpy
(
dist
,
p_dist
,
sizeof
(
float
)
*
nq
*
k
);
}
catch
(
KnowhereException
&
e
)
{
WRAPPER_LOG_ERROR
<<
e
.
what
();
return
KNOWHERE_UNEXPECTED_ERROR
;
...
...
cpp/src/wrapper/knowhere/vec_index.cpp
浏览文件 @
9af8a46f
...
...
@@ -14,6 +14,8 @@
#include "vec_impl.h"
#include "wrapper_log.h"
#include <cuda.h>
namespace
zilliz
{
namespace
milvus
{
...
...
@@ -246,11 +248,13 @@ void ParameterValidation(const IndexType &type, Config &cfg) {
case
IndexType
::
FAISS_IVFSQ8_GPU
:
case
IndexType
::
FAISS_IVFFLAT_GPU
:
case
IndexType
::
FAISS_IVFPQ_GPU
:
{
//search on GPU
if
(
cfg
.
get_with_default
(
"nprobe"
,
0
)
!=
0
)
{
auto
nprobe
=
cfg
[
"nprobe"
].
as
<
int
>
();
if
(
nprobe
>
GPU_MAX_NRPOBE
)
{
WRAPPER_LOG_WARNING
<<
"When search with GPU, nprobe shoud be no more than "
<<
GPU_MAX_NRPOBE
<<
", but you passed "
<<
nprobe
<<
". Search with "
<<
GPU_MAX_NRPOBE
<<
" instead"
;
WRAPPER_LOG_WARNING
<<
"When search with GPU, nprobe shoud be no more than "
<<
GPU_MAX_NRPOBE
<<
", but you passed "
<<
nprobe
<<
". Search with "
<<
GPU_MAX_NRPOBE
<<
" instead"
;
cfg
.
insert_or_assign
(
"nprobe"
,
GPU_MAX_NRPOBE
);
}
}
...
...
cpp/src/wrapper/knowhere/vec_index.h
浏览文件 @
9af8a46f
...
...
@@ -14,8 +14,6 @@
#include "knowhere/common/config.h"
#include "knowhere/common/binary_set.h"
#include "cuda.h"
namespace
zilliz
{
namespace
milvus
{
...
...
@@ -62,7 +60,7 @@ class VecIndex {
long
*
ids
,
const
Config
&
cfg
=
Config
())
=
0
;
virtual
VecIndexPtr
CopyToGpu
(
const
int64_t
&
device_id
,
virtual
VecIndexPtr
CopyToGpu
(
const
int64_t
&
device_id
,
const
Config
&
cfg
=
Config
())
=
0
;
virtual
VecIndexPtr
CopyToCpu
(
const
Config
&
cfg
=
Config
())
=
0
;
...
...
@@ -86,16 +84,16 @@ extern ErrorCode write_index(VecIndexPtr index, const std::string &location);
extern
VecIndexPtr
read_index
(
const
std
::
string
&
location
);
extern
VecIndexPtr
GetVecIndexFactory
(
const
IndexType
&
type
,
const
Config
&
cfg
=
Config
());
extern
VecIndexPtr
GetVecIndexFactory
(
const
IndexType
&
type
,
const
Config
&
cfg
=
Config
());
extern
VecIndexPtr
LoadVecIndex
(
const
IndexType
&
index_type
,
const
zilliz
::
knowhere
::
BinarySet
&
index_binary
);
extern
void
AutoGenParams
(
const
IndexType
&
type
,
const
long
&
size
,
Config
&
cfg
);
extern
void
AutoGenParams
(
const
IndexType
&
type
,
const
long
&
size
,
Config
&
cfg
);
extern
void
ParameterValidation
(
const
IndexType
&
type
,
Config
&
cfg
);
extern
void
ParameterValidation
(
const
IndexType
&
type
,
Config
&
cfg
);
extern
IndexType
ConvertToCpuIndexType
(
const
IndexType
&
type
);
extern
IndexType
ConvertToGpuIndexType
(
const
IndexType
&
type
);
extern
IndexType
ConvertToCpuIndexType
(
const
IndexType
&
type
);
extern
IndexType
ConvertToGpuIndexType
(
const
IndexType
&
type
);
}
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录