Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
milvus
提交
539ce693
milvus
项目概览
BaiXuePrincess
/
milvus
与 Fork 源项目一致
Fork自
milvus / milvus
通知
7
Star
4
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
539ce693
编写于
3月 11, 2020
作者:
Y
yudong.cai
浏览文件
操作
浏览文件
下载
差异文件
Merge remote-tracking branch 'main/master' into caiyd_1547_rename_storage_file
上级
98da92af
40c7e8ba
变更
2
隐藏空白更改
内联
并排
Showing
2 changed file
with
31 addition
and
22 deletion
+31
-22
CHANGELOG.md
CHANGELOG.md
+1
-1
core/src/db/insert/MemTable.cpp
core/src/db/insert/MemTable.cpp
+30
-21
未找到文件。
CHANGELOG.md
浏览文件 @
539ce693
...
...
@@ -40,7 +40,7 @@ Please mark all change in change log and use the issue from GitHub
-
\#
1507 set_config for insert_buffer_size is wrong
-
\#
1510 Add set interfaces for WAL configurations
-
\#
1511 Fix big integer cannot pass to server correctly
-
\#
1517
r
esult is not correct when search vectors in multi partition, index type is RNSG
-
\#
1517
R
esult is not correct when search vectors in multi partition, index type is RNSG
-
\#
1518 Table count did not match after deleting vectors and compact
-
\#
1521 Make cache_insert_data take effect in-service
-
\#
1525 Add setter API for config preload_table
...
...
core/src/db/insert/MemTable.cpp
浏览文件 @
539ce693
...
...
@@ -179,7 +179,7 @@ MemTable::ApplyDeletes() {
auto
start_total
=
std
::
chrono
::
high_resolution_clock
::
now
();
auto
start
=
std
::
chrono
::
high_resolution_clock
::
now
();
//
auto start = std::chrono::high_resolution_clock::now();
std
::
vector
<
int
>
file_types
{
meta
::
TableFileSchema
::
FILE_TYPE
::
RAW
,
meta
::
TableFileSchema
::
FILE_TYPE
::
TO_INDEX
,
meta
::
TableFileSchema
::
FILE_TYPE
::
BACKUP
};
...
...
@@ -220,9 +220,9 @@ MemTable::ApplyDeletes() {
OngoingFileChecker
::
GetInstance
().
MarkOngoingFiles
(
files_to_check
);
auto
end
=
std
::
chrono
::
high_resolution_clock
::
now
();
std
::
chrono
::
duration
<
double
>
diff
=
end
-
start
;
ENGINE_LOG_DEBUG
<<
"Found "
<<
ids_to_check_map
.
size
()
<<
" segment to apply deletes in "
<<
diff
.
count
()
<<
" s"
;
auto
time0
=
std
::
chrono
::
high_resolution_clock
::
now
();
std
::
chrono
::
duration
<
double
>
diff
0
=
time0
-
start_total
;
ENGINE_LOG_DEBUG
<<
"Found "
<<
ids_to_check_map
.
size
()
<<
" segment to apply deletes in "
<<
diff
0
.
count
()
<<
" s"
;
meta
::
TableFilesSchema
table_files_to_update
;
...
...
@@ -230,7 +230,7 @@ MemTable::ApplyDeletes() {
auto
&
table_file
=
table_files
[
kv
.
first
];
ENGINE_LOG_DEBUG
<<
"Applying deletes in segment: "
<<
table_file
.
segment_id_
;
start
=
std
::
chrono
::
high_resolution_clock
::
now
();
auto
time1
=
std
::
chrono
::
high_resolution_clock
::
now
();
std
::
string
segment_dir
;
utils
::
GetParentPath
(
table_file
.
location_
,
segment_dir
);
...
...
@@ -274,17 +274,15 @@ MemTable::ApplyDeletes() {
segment
::
DeletedDocsPtr
deleted_docs
=
std
::
make_shared
<
segment
::
DeletedDocs
>
();
end
=
std
::
chrono
::
high_resolution_clock
::
now
();
diff
=
end
-
start
;
ENGINE_LOG_DEBUG
<<
"Loading uids and deleted docs took "
<<
diff
.
count
()
<<
" s"
;
start
=
std
::
chrono
::
high_resolution_clock
::
now
();
auto
time2
=
std
::
chrono
::
high_resolution_clock
::
now
();
std
::
chrono
::
duration
<
double
>
diff1
=
time2
-
time1
;
ENGINE_LOG_DEBUG
<<
"Loading uids and deleted docs took "
<<
diff1
.
count
()
<<
" s"
;
std
::
sort
(
ids_to_check
.
begin
(),
ids_to_check
.
end
());
end
=
std
::
chrono
::
high_resolution_clock
::
now
();
diff
=
end
-
start
;
ENGINE_LOG_DEBUG
<<
"Sorting "
<<
ids_to_check
.
size
()
<<
" ids took "
<<
diff
.
count
()
<<
" s"
;
auto
time3
=
std
::
chrono
::
high_resolution_clock
::
now
();
std
::
chrono
::
duration
<
double
>
diff2
=
time3
-
time2
;
ENGINE_LOG_DEBUG
<<
"Sorting "
<<
ids_to_check
.
size
()
<<
" ids took "
<<
diff
2
.
count
()
<<
" s"
;
size_t
delete_count
=
0
;
auto
find_diff
=
std
::
chrono
::
duration
<
double
>::
zero
();
...
...
@@ -324,11 +322,13 @@ MemTable::ApplyDeletes() {
<<
find_diff
.
count
()
<<
" s in total"
;
ENGINE_LOG_DEBUG
<<
"Setting deleted docs and bloom filter took "
<<
set_diff
.
count
()
<<
" s in total"
;
auto
time4
=
std
::
chrono
::
high_resolution_clock
::
now
();
for
(
auto
i
=
0
;
i
<
indexes
.
size
();
++
i
)
{
indexes
[
i
]
->
SetBlacklist
(
blacklists
[
i
]);
}
start
=
std
::
chrono
::
high_resolution_clock
::
now
();
//
start = std::chrono::high_resolution_clock::now();
segment
::
Segment
tmp_segment
;
segment
::
SegmentWriter
segment_writer
(
segment_dir
);
...
...
@@ -337,21 +337,21 @@ MemTable::ApplyDeletes() {
break
;
}
end
=
std
::
chrono
::
high_resolution_clock
::
now
();
diff
=
end
-
start
;
auto
time5
=
std
::
chrono
::
high_resolution_clock
::
now
();
std
::
chrono
::
duration
<
double
>
diff4
=
time5
-
time4
;
ENGINE_LOG_DEBUG
<<
"Appended "
<<
deleted_docs
->
GetSize
()
<<
" offsets to deleted docs in segment: "
<<
table_file
.
segment_id_
<<
" in "
<<
diff
.
count
()
<<
" offsets to deleted docs in segment: "
<<
table_file
.
segment_id_
<<
" in "
<<
diff
4
.
count
()
<<
" s"
;
start
=
std
::
chrono
::
high_resolution_clock
::
now
();
//
start = std::chrono::high_resolution_clock::now();
status
=
segment_writer
.
WriteBloomFilter
(
id_bloom_filter_ptr
);
if
(
!
status
.
ok
())
{
break
;
}
end
=
std
::
chrono
::
high_resolution_clock
::
now
();
diff
=
end
-
start
;
ENGINE_LOG_DEBUG
<<
"Updated bloom filter in segment: "
<<
table_file
.
segment_id_
<<
" in "
<<
diff
.
count
()
auto
time6
=
std
::
chrono
::
high_resolution_clock
::
now
();
std
::
chrono
::
duration
<
double
>
diff5
=
time6
-
time5
;
ENGINE_LOG_DEBUG
<<
"Updated bloom filter in segment: "
<<
table_file
.
segment_id_
<<
" in "
<<
diff
5
.
count
()
<<
" s"
;
// Update table file row count
...
...
@@ -362,8 +362,15 @@ MemTable::ApplyDeletes() {
table_files_to_update
.
emplace_back
(
file
);
}
}
auto
time7
=
std
::
chrono
::
high_resolution_clock
::
now
();
std
::
chrono
::
duration
<
double
>
diff6
=
time7
-
time6
;
diff6
=
time6
-
time5
;
ENGINE_LOG_DEBUG
<<
"Update table file row count in vector of segment: "
<<
table_file
.
segment_id_
<<
" in "
<<
diff6
.
count
()
<<
" s"
;
}
auto
time7
=
std
::
chrono
::
high_resolution_clock
::
now
();
status
=
meta_
->
UpdateTableFilesRowCount
(
table_files_to_update
);
if
(
!
status
.
ok
())
{
...
...
@@ -375,6 +382,8 @@ MemTable::ApplyDeletes() {
doc_ids_to_delete_
.
clear
();
auto
end_total
=
std
::
chrono
::
high_resolution_clock
::
now
();
std
::
chrono
::
duration
<
double
>
diff7
=
end_total
-
time7
;
ENGINE_LOG_DEBUG
<<
"Update deletes to meta in table "
<<
table_id_
<<
" in "
<<
diff7
.
count
()
<<
" s"
;
std
::
chrono
::
duration
<
double
>
diff_total
=
end_total
-
start_total
;
ENGINE_LOG_DEBUG
<<
"Finished applying deletes in table "
<<
table_id_
<<
" in "
<<
diff_total
.
count
()
<<
" s"
;
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录