Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
milvus
提交
2f020329
milvus
项目概览
BaiXuePrincess
/
milvus
与 Fork 源项目一致
从无法访问的项目Fork
通知
7
Star
4
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
2f020329
编写于
8月 30, 2019
作者:
G
groot
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
MS-443 Create index hang again
Former-commit-id: 6f27554be8f91c8a5b7c01c715fd64ff22f3a8b8
上级
d2d20b51
变更
3
显示空白变更内容
内联
并排
Showing
3 changed file
with
42 addition
and
4 deletion
+42
-4
cpp/CHANGELOG.md
cpp/CHANGELOG.md
+1
-0
cpp/src/db/DBImpl.cpp
cpp/src/db/DBImpl.cpp
+3
-0
cpp/src/db/engine/ExecutionEngineImpl.cpp
cpp/src/db/engine/ExecutionEngineImpl.cpp
+38
-4
未找到文件。
cpp/CHANGELOG.md
浏览文件 @
2f020329
...
...
@@ -15,6 +15,7 @@ Please mark all change in change log and use the ticket from JIRA.
-
MS-431 - Search vectors params nprobe: 0/-1, expected result: raise exception
-
MS-331 - Crate Table : when table exists, error code is META_FAILED(code=15) rather than ILLEGAL TABLE NAME(code=9))
-
MS-430 - Search no result if index created with FLAT
-
MS-443 - Create index hang again
## Improvement
-
MS-327 - Clean code for milvus
...
...
cpp/src/db/DBImpl.cpp
浏览文件 @
2f020329
...
...
@@ -581,6 +581,9 @@ Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index)
while
(
!
file_ids
.
empty
())
{
ENGINE_LOG_DEBUG
<<
"Non index files detected! Will build index "
<<
times
;
if
(
index
.
engine_type_
!=
(
int
)
EngineType
::
FAISS_IDMAP
)
{
status
=
meta_ptr_
->
UpdateTableFilesToIndex
(
table_id
);
}
std
::
this_thread
::
sleep_for
(
std
::
chrono
::
milliseconds
(
std
::
min
(
10
*
1000
,
times
*
100
)));
status
=
meta_ptr_
->
FilesByType
(
table_id
,
file_types
,
file_ids
);
...
...
cpp/src/db/engine/ExecutionEngineImpl.cpp
浏览文件 @
2f020329
...
...
@@ -93,7 +93,7 @@ Status ExecutionEngineImpl::AddWithIds(long n, const float *xdata, const long *x
size_t
ExecutionEngineImpl
::
Count
()
const
{
if
(
index_
==
nullptr
)
{
ENGINE_LOG_ERROR
<<
"ExecutionEngineImpl:
:index is null
"
;
ENGINE_LOG_ERROR
<<
"ExecutionEngineImpl:
index is null, return count 0
"
;
return
0
;
}
return
index_
->
Count
();
...
...
@@ -105,7 +105,7 @@ size_t ExecutionEngineImpl::Size() const {
size_t
ExecutionEngineImpl
::
Dimension
()
const
{
if
(
index_
==
nullptr
)
{
ENGINE_LOG_ERROR
<<
"ExecutionEngineImpl:
:index is null"
;
ENGINE_LOG_ERROR
<<
"ExecutionEngineImpl:
index is null, return dimension "
<<
dim_
;
return
dim_
;
}
return
index_
->
Dimension
();
...
...
@@ -126,12 +126,16 @@ Status ExecutionEngineImpl::Serialize() {
Status
ExecutionEngineImpl
::
Load
(
bool
to_cache
)
{
index_
=
zilliz
::
milvus
::
cache
::
CpuCacheMgr
::
GetInstance
()
->
GetIndex
(
location_
);
bool
already_in_cache
=
(
index_
!=
nullptr
);
if
(
!
index_
)
{
if
(
!
already_in_cache
)
{
try
{
double
physical_size
=
PhysicalSize
();
server
::
CollectExecutionEngineMetrics
metrics
(
physical_size
);
index_
=
read_index
(
location_
);
if
(
index_
==
nullptr
)
{
ENGINE_LOG_ERROR
<<
"Failed to load index from "
<<
location_
;
}
else
{
ENGINE_LOG_DEBUG
<<
"Disk io from: "
<<
location_
;
}
}
catch
(
knowhere
::
KnowhereException
&
e
)
{
ENGINE_LOG_ERROR
<<
e
.
what
();
return
Status
::
Error
(
e
.
what
());
...
...
@@ -152,6 +156,11 @@ Status ExecutionEngineImpl::CopyToGpu(uint64_t device_id) {
if
(
already_in_cache
)
{
index_
=
index
;
}
else
{
if
(
index_
==
nullptr
)
{
ENGINE_LOG_ERROR
<<
"ExecutionEngineImpl: index is null, failed to copy to gpu"
;
return
Status
::
Error
(
"index is null"
);
}
try
{
index_
=
index_
->
CopyToGpu
(
device_id
);
ENGINE_LOG_DEBUG
<<
"CPU to GPU"
<<
device_id
;
...
...
@@ -176,6 +185,11 @@ Status ExecutionEngineImpl::CopyToCpu() {
if
(
already_in_cache
)
{
index_
=
index
;
}
else
{
if
(
index_
==
nullptr
)
{
ENGINE_LOG_ERROR
<<
"ExecutionEngineImpl: index is null, failed to copy to cpu"
;
return
Status
::
Error
(
"index is null"
);
}
try
{
index_
=
index_
->
CopyToCpu
();
ENGINE_LOG_DEBUG
<<
"GPU to CPU"
;
...
...
@@ -194,6 +208,11 @@ Status ExecutionEngineImpl::CopyToCpu() {
}
ExecutionEnginePtr
ExecutionEngineImpl
::
Clone
()
{
if
(
index_
==
nullptr
)
{
ENGINE_LOG_ERROR
<<
"ExecutionEngineImpl: index is null, failed to clone"
;
return
nullptr
;
}
auto
ret
=
std
::
make_shared
<
ExecutionEngineImpl
>
(
dim_
,
location_
,
index_type_
,
metric_type_
,
nlist_
);
ret
->
Init
();
ret
->
index_
=
index_
->
Clone
();
...
...
@@ -220,6 +239,11 @@ Status ExecutionEngineImpl::Merge(const std::string &location) {
}
}
if
(
index_
==
nullptr
)
{
ENGINE_LOG_ERROR
<<
"ExecutionEngineImpl: index is null, failed to merge"
;
return
Status
::
Error
(
"index is null"
);
}
if
(
auto
file_index
=
std
::
dynamic_pointer_cast
<
BFIndex
>
(
to_merge
))
{
auto
ec
=
index_
->
Add
(
file_index
->
Count
(),
file_index
->
GetRawVectors
(),
file_index
->
GetRawIds
());
if
(
ec
!=
server
::
KNOWHERE_SUCCESS
)
{
...
...
@@ -237,6 +261,11 @@ ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_t
ENGINE_LOG_DEBUG
<<
"Build index file: "
<<
location
<<
" from: "
<<
location_
;
auto
from_index
=
std
::
dynamic_pointer_cast
<
BFIndex
>
(
index_
);
if
(
from_index
==
nullptr
)
{
ENGINE_LOG_ERROR
<<
"ExecutionEngineImpl: from_index is null, failed to build index"
;
return
nullptr
;
}
auto
to_index
=
CreatetVecIndex
(
engine_type
);
if
(
!
to_index
)
{
throw
Exception
(
"Create Empty VecIndex"
);
...
...
@@ -264,6 +293,11 @@ Status ExecutionEngineImpl::Search(long n,
long
nprobe
,
float
*
distances
,
long
*
labels
)
const
{
if
(
index_
==
nullptr
)
{
ENGINE_LOG_ERROR
<<
"ExecutionEngineImpl: index is null, failed to search"
;
return
Status
::
Error
(
"index is null"
);
}
ENGINE_LOG_DEBUG
<<
"Search Params: [k] "
<<
k
<<
" [nprobe] "
<<
nprobe
;
auto
ec
=
index_
->
Search
(
n
,
data
,
distances
,
labels
,
Config
::
object
{{
"k"
,
k
},
{
"nprobe"
,
nprobe
}});
if
(
ec
!=
server
::
KNOWHERE_SUCCESS
)
{
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录