Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
milvus
提交
c2b5b005
milvus
项目概览
BaiXuePrincess
/
milvus
与 Fork 源项目一致
Fork自
milvus / milvus
通知
7
Star
4
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
前往新版Gitcode,体验更适合开发者的 AI 搜索 >>
提交
c2b5b005
编写于
3月 07, 2020
作者:
G
groot
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
#1532 Search with ivf_flat failed with open-dataset: sift-256-hamming
Signed-off-by:
N
groot
<
yihua.mo@zilliz.com
>
上级
a08b51c2
变更
7
隐藏空白更改
内联
并排
Showing
7 changed file
with
16 addition
and
9 deletion
+16
-9
CHANGELOG.md
CHANGELOG.md
+1
-0
core/src/db/DBImpl.cpp
core/src/db/DBImpl.cpp
+4
-4
core/src/db/Utils.cpp
core/src/db/Utils.cpp
+5
-0
core/src/db/Utils.h
core/src/db/Utils.h
+3
-0
core/src/db/engine/ExecutionEngineImpl.cpp
core/src/db/engine/ExecutionEngineImpl.cpp
+1
-1
core/src/db/meta/MySQLMetaImpl.cpp
core/src/db/meta/MySQLMetaImpl.cpp
+1
-2
core/src/db/meta/SqliteMetaImpl.cpp
core/src/db/meta/SqliteMetaImpl.cpp
+1
-2
未找到文件。
CHANGELOG.md
浏览文件 @
c2b5b005
...
...
@@ -40,6 +40,7 @@ Please mark all change in change log and use the issue from GitHub
-
\#
1521 Make cache_insert_data take effect in-service
-
\#
1525 Add setter API for config preload_table
-
\#
1530 Set table file with correct engine type in meta
-
\#
1532 Search with ivf_flat failed with open-dataset: sift-256-hamming
-
\#
1535 Degradation searching performance with metric_type: binary_idmap
## Feature
...
...
core/src/db/DBImpl.cpp
浏览文件 @
c2b5b005
...
...
@@ -812,7 +812,7 @@ DBImpl::CompactFile(const std::string& table_id, const meta::TableFileSchema& fi
// Update table files state
// if index type isn't IDMAP, set file type to TO_INDEX if file size exceed index_file_size
// else set file type to RAW, no need to build index
if
(
compacted_file
.
engine_type_
!=
(
int
)
EngineType
::
FAISS_IDMAP
)
{
if
(
!
utils
::
IsRawIndexType
(
compacted_file
.
engine_type_
)
)
{
compacted_file
.
file_type_
=
(
segment_writer_ptr
->
Size
()
>=
compacted_file
.
index_file_size_
)
?
meta
::
TableFileSchema
::
TO_INDEX
:
meta
::
TableFileSchema
::
RAW
;
...
...
@@ -1468,7 +1468,7 @@ DBImpl::MergeFiles(const std::string& table_id, const meta::TableFilesSchema& fi
// step 4: update table files state
// if index type isn't IDMAP, set file type to TO_INDEX if file size exceed index_file_size
// else set file type to RAW, no need to build index
if
(
table_file
.
engine_type_
!=
(
int
)
EngineType
::
FAISS_IDMAP
)
{
if
(
!
utils
::
IsRawIndexType
(
table_file
.
engine_type_
)
)
{
table_file
.
file_type_
=
(
segment_writer_ptr
->
Size
()
>=
table_file
.
index_file_size_
)
?
meta
::
TableFileSchema
::
TO_INDEX
:
meta
::
TableFileSchema
::
RAW
;
...
...
@@ -1770,7 +1770,7 @@ DBImpl::BuildTableIndexRecursively(const std::string& table_id, const TableIndex
// for IDMAP type, only wait all NEW file converted to RAW file
// for other type, wait NEW/RAW/NEW_MERGE/NEW_INDEX/TO_INDEX files converted to INDEX files
std
::
vector
<
int
>
file_types
;
if
(
index
.
engine_type_
==
static_cast
<
int32_t
>
(
EngineType
::
FAISS_IDMAP
))
{
if
(
utils
::
IsRawIndexType
(
index
.
engine_type_
))
{
file_types
=
{
static_cast
<
int32_t
>
(
meta
::
TableFileSchema
::
NEW
),
static_cast
<
int32_t
>
(
meta
::
TableFileSchema
::
NEW_MERGE
),
...
...
@@ -1792,7 +1792,7 @@ DBImpl::BuildTableIndexRecursively(const std::string& table_id, const TableIndex
while
(
!
table_files
.
empty
())
{
ENGINE_LOG_DEBUG
<<
"Non index files detected! Will build index "
<<
times
;
if
(
index
.
engine_type_
!=
(
int
)
EngineType
::
FAISS_IDMAP
)
{
if
(
!
utils
::
IsRawIndexType
(
index
.
engine_type_
)
)
{
status
=
meta_ptr_
->
UpdateTableFilesToIndex
(
table_id
);
}
...
...
core/src/db/Utils.cpp
浏览文件 @
c2b5b005
...
...
@@ -215,6 +215,11 @@ IsSameIndex(const TableIndex& index1, const TableIndex& index2) {
index1
.
metric_type_
==
index2
.
metric_type_
;
}
bool
IsRawIndexType
(
int32_t
type
)
{
return
(
type
==
(
int32_t
)
EngineType
::
FAISS_IDMAP
)
||
(
type
==
(
int32_t
)
EngineType
::
FAISS_BIN_IDMAP
);
}
meta
::
DateT
GetDate
(
const
std
::
time_t
&
t
,
int
day_delta
)
{
struct
tm
ltm
;
...
...
core/src/db/Utils.h
浏览文件 @
c2b5b005
...
...
@@ -45,6 +45,9 @@ GetParentPath(const std::string& path, std::string& parent_path);
bool
IsSameIndex
(
const
TableIndex
&
index1
,
const
TableIndex
&
index2
);
bool
IsRawIndexType
(
int32_t
type
);
meta
::
DateT
GetDate
(
const
std
::
time_t
&
t
,
int
day_delta
=
0
);
meta
::
DateT
...
...
core/src/db/engine/ExecutionEngineImpl.cpp
浏览文件 @
c2b5b005
...
...
@@ -406,7 +406,7 @@ ExecutionEngineImpl::Load(bool to_cache) {
utils
::
GetParentPath
(
location_
,
segment_dir
);
auto
segment_reader_ptr
=
std
::
make_shared
<
segment
::
SegmentReader
>
(
segment_dir
);
if
(
index_type_
==
EngineType
::
FAISS_IDMAP
||
index_type_
==
EngineType
::
FAISS_BIN_IDMAP
)
{
if
(
utils
::
IsRawIndexType
((
int32_t
)
index_type_
)
)
{
index_
=
index_type_
==
EngineType
::
FAISS_IDMAP
?
GetVecIndexFactory
(
IndexType
::
FAISS_IDMAP
)
:
GetVecIndexFactory
(
IndexType
::
FAISS_BIN_IDMAP
);
milvus
::
json
conf
{{
knowhere
::
meta
::
DEVICEID
,
gpu_num_
},
{
knowhere
::
meta
::
DIM
,
dim_
}};
...
...
core/src/db/meta/MySQLMetaImpl.cpp
浏览文件 @
c2b5b005
...
...
@@ -2086,8 +2086,7 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/)
// If we are deleting a raw table file, it means it's okay to delete the entire segment directory.
// Else, we can only delete the single file
// TODO(zhiru): We determine whether a table file is raw by its engine type. This is a bit hacky
if
(
table_file
.
engine_type_
==
(
int32_t
)
EngineType
::
FAISS_IDMAP
||
table_file
.
engine_type_
==
(
int32_t
)
EngineType
::
FAISS_BIN_IDMAP
)
{
if
(
utils
::
IsRawIndexType
(
table_file
.
engine_type_
))
{
utils
::
DeleteSegment
(
options_
,
table_file
);
std
::
string
segment_dir
;
utils
::
GetParentPath
(
table_file
.
location_
,
segment_dir
);
...
...
core/src/db/meta/SqliteMetaImpl.cpp
浏览文件 @
c2b5b005
...
...
@@ -1425,8 +1425,7 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/
// If we are deleting a raw table file, it means it's okay to delete the entire segment directory.
// Else, we can only delete the single file
// TODO(zhiru): We determine whether a table file is raw by its engine type. This is a bit hacky
if
(
table_file
.
engine_type_
==
(
int32_t
)
EngineType
::
FAISS_IDMAP
||
table_file
.
engine_type_
==
(
int32_t
)
EngineType
::
FAISS_BIN_IDMAP
)
{
if
(
utils
::
IsRawIndexType
(
table_file
.
engine_type_
))
{
utils
::
DeleteSegment
(
options_
,
table_file
);
std
::
string
segment_dir
;
utils
::
GetParentPath
(
table_file
.
location_
,
segment_dir
);
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录