Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
indiff7643
Terarkdb
提交
a4d77e92
T
Terarkdb
项目概览
indiff7643
/
Terarkdb
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
Terarkdb
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
a4d77e92
编写于
1月 11, 2021
作者:
U
ustcwelcome
提交者:
奏之章
2月 08, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
bug-fix of ttl on compaction
上级
a726402c
变更
6
隐藏空白更改
内联
并排
Showing
6 changed file
with
52 addition
and
28 deletion
+52
-28
db/compaction_job.cc
db/compaction_job.cc
+8
-0
db/db_impl.cc
db/db_impl.cc
+4
-5
db/db_impl_gc_ttl_test.cc
db/db_impl_gc_ttl_test.cc
+3
-4
table/block_based_table_builder.cc
table/block_based_table_builder.cc
+5
-6
table/meta_blocks.cc
table/meta_blocks.cc
+18
-2
table/table_properties.cc
table/table_properties.cc
+14
-11
未找到文件。
db/compaction_job.cc
浏览文件 @
a4d77e92
...
...
@@ -891,6 +891,9 @@ Status CompactionJob::Run() {
output
.
meta
.
prop
.
inheritance_chain
=
tp
->
inheritance_chain
;
output
.
meta
.
prop
.
ratio_expire_time
=
tp
->
ratio_expire_time
;
output
.
meta
.
prop
.
scan_gap_expire_time
=
tp
->
scan_gap_expire_time
;
// ROCKS_LOG_INFO(db_options_.info_log,
// "ratio:%" PRIu64 ", scan:%" PRIu64,
// tp->ratio_expire_time, tp->scan_gap_expire_time);
output
.
finished
=
true
;
c
->
AddOutputTableFileNumber
(
file_number
);
}
...
...
@@ -2117,6 +2120,11 @@ Status CompactionJob::FinishCompactionOutputFile(
tp
.
num_range_deletions
>
0
?
0
:
TablePropertyCache
::
kNoRangeDeletions
;
meta
->
prop
.
flags
|=
tp
.
snapshots
.
empty
()
?
0
:
TablePropertyCache
::
kHasSnapshots
;
meta
->
prop
.
ratio_expire_time
=
tp
.
ratio_expire_time
;
meta
->
prop
.
scan_gap_expire_time
=
tp
.
scan_gap_expire_time
;
ROCKS_LOG_INFO
(
db_options_
.
info_log
,
"ratio:%"
PRIu64
", scan:%"
PRIu64
,
tp
.
ratio_expire_time
,
tp
.
scan_gap_expire_time
);
}
if
(
s
.
ok
()
&&
tp
.
num_entries
==
0
&&
tp
.
num_range_deletions
==
0
)
{
...
...
db/db_impl.cc
浏览文件 @
a4d77e92
...
...
@@ -17,6 +17,7 @@
#endif
#include <algorithm>
#include <cinttypes>
#include <cstdio>
#include <map>
#include <set>
...
...
@@ -929,12 +930,10 @@ void DBImpl::ScheduleGCTTL() {
uint64_t
marked_count
=
0
;
uint64_t
nowSeconds
=
env_
->
NowMicros
()
/
1000U
/
1000U
;
auto
should_marked_for_compacted
=
[
&
](
uint64_t
ratio_expire_time
,
uint64_t
scan_gap_expire_time
,
uint64_t
now
)
{
uint64_t
scan_gap_expire_time
,
uint64_t
now
)
{
ROCKS_LOG_INFO
(
immutable_db_options_
.
info_log
,
"SST Table property info: % PRIu64 "
",% PRIu64 "
",% PRIu64 "
,
"SST Table property info:%"
PRIu64
",%"
PRIu64
",%"
PRIu64
,
ratio_expire_time
,
scan_gap_expire_time
,
now
);
return
(
std
::
min
(
ratio_expire_time
,
scan_gap_expire_time
)
<=
now
);
};
...
...
db/db_impl_gc_ttl_test.cc
浏览文件 @
a4d77e92
...
...
@@ -24,6 +24,7 @@ class DBImplGCTTL_Test : public DBTestBase {
options
.
ttl_scan_gap
=
10
;
options
.
ttl_extractor_factory
.
reset
(
new
test
::
TestTtlExtractorFactory
());
options
.
level0_file_num_compaction_trigger
=
8
;
options
.
enable_lazy_compaction
=
false
;
options
.
table_factory
.
reset
(
new
BlockBasedTableFactory
(
BlockBasedTableOptions
()));
}
...
...
@@ -50,9 +51,7 @@ class DBImplGCTTL_Test : public DBTestBase {
flag
=
true
;
});
rocksdb
::
SyncPoint
::
GetInstance
()
->
SetCallBack
(
"DBImpl:ScheduleGCTTL-mark"
,
[
&
](
void
*
/*arg*/
)
{
mark
++
;
});
"DBImpl:ScheduleGCTTL-mark"
,
[
&
](
void
*
/*arg*/
)
{
mark
++
;
});
}
};
...
...
@@ -81,7 +80,7 @@ TEST_F(DBImplGCTTL_Test, L0FileExpiredTest) {
dbfull
()
->
TEST_WaitForStatsDumpRun
([
&
]
{
mock_env_
->
set_current_time
(
ttl
);
});
ASSERT_TRUE
(
flag
);
ASSERT_EQ
(
L0FilesNums
,
mark
);
dbfull
()
->
CompactRange
(
CompactRangeOptions
(),
nullptr
,
nullptr
);
dbfull
()
->
CompactRange
(
CompactRangeOptions
(),
nullptr
,
nullptr
);
dbfull
()
->
TEST_WaitForCompact
();
dbfull
()
->
ScheduleGCTTL
();
...
...
table/block_based_table_builder.cc
浏览文件 @
a4d77e92
...
...
@@ -842,9 +842,9 @@ void BlockBasedTableBuilder::WritePropertiesBlock(
uint64_t
max_uint64_t
=
std
::
numeric_limits
<
uint64_t
>::
max
();
double
ratio
=
rep_
->
moptions
.
ttl_garbage_collection_percentage
;
uint64_t
percentile_ratio_ttl
=
max_uint64_t
;
if
(
ratio
<=
100.0
)
{
assert
(
ttl_histogram_
!=
nullptr
);
uint64_t
percentile_ratio_ttl
=
max_uint64_t
;
if
(
!
ttl_histogram_
->
Empty
()
&&
kv_size_has_row_ttl_
>=
...
...
@@ -870,9 +870,8 @@ void BlockBasedTableBuilder::WritePropertiesBlock(
ROCKS_LOG_INFO
(
rep_
->
ioptions
.
info_log
,
"[%s] ratio_row_ttl:%"
PRIu64
", scan_gap_row_ttl:%"
PRIu64
"."
,
rep_
->
column_family_name
.
c_str
(),
rep_
->
props
.
ratio_expire_time
-
now_seconds
,
rep_
->
props
.
scan_gap_expire_time
-
now_seconds
);
rep_
->
column_family_name
.
c_str
(),
percentile_ratio_ttl
,
min_ttl_seconds_
);
min_ttl_seconds_
=
std
::
numeric_limits
<
uint64_t
>::
max
();
ttl_histogram_
.
reset
();
ttl_extractor_
.
reset
();
...
...
@@ -935,8 +934,8 @@ Status BlockBasedTableBuilder::Finish(
r
->
props
.
read_amp
=
prop
->
read_amp
;
r
->
props
.
dependence
=
prop
->
dependence
;
r
->
props
.
inheritance_chain
=
prop
->
inheritance_chain
;
r
->
props
.
ratio_expire_time
=
prop
->
ratio_expire_time
;
r
->
props
.
scan_gap_expire_time
=
prop
->
scan_gap_expire_time
;
//
r->props.ratio_expire_time = prop->ratio_expire_time;
//
r->props.scan_gap_expire_time = prop->scan_gap_expire_time;
}
if
(
snapshots
!=
nullptr
)
{
r
->
props
.
snapshots
=
*
snapshots
;
...
...
table/meta_blocks.cc
浏览文件 @
a4d77e92
...
...
@@ -10,6 +10,7 @@
#include "db/table_properties_collector.h"
#include "rocksdb/table.h"
#include "rocksdb/table_properties.h"
#include "rocksdb/terark_namespace.h"
#include "table/block.h"
#include "table/block_fetcher.h"
#include "table/format.h"
...
...
@@ -19,7 +20,6 @@
#include "util/coding.h"
#include "util/file_reader_writer.h"
#include "rocksdb/terark_namespace.h"
namespace
TERARKDB_NAMESPACE
{
MetaIndexBuilder
::
MetaIndexBuilder
()
...
...
@@ -157,9 +157,11 @@ void PropertyBlockBuilder::AddTableProperty(const TableProperties& props) {
if
(
!
props
.
compression_name
.
empty
())
{
Add
(
TablePropertiesNames
::
kCompression
,
props
.
compression_name
);
}
if
(
true
)
{
if
(
props
.
ratio_expire_time
<
std
::
numeric_limits
<
uint64_t
>::
max
()
)
{
Add
(
TablePropertiesNames
::
kEarliestTimeBeginCompact
,
props
.
ratio_expire_time
);
}
if
(
props
.
scan_gap_expire_time
<
std
::
numeric_limits
<
uint64_t
>::
max
())
{
Add
(
TablePropertiesNames
::
kLatestTimeEndCompact
,
props
.
scan_gap_expire_time
);
}
...
...
@@ -424,6 +426,20 @@ Status ReadProperties(const Slice& handle_value, RandomAccessFileReader* file,
}
}
else
if
(
key
==
TablePropertiesNames
::
kInheritanceChain
)
{
GetUint64Vector
(
key
,
&
raw_val
,
new_table_properties
->
inheritance_chain
);
}
else
if
(
key
==
TablePropertiesNames
::
kRatioExpireTime
)
{
uint64_t
u64_val
;
if
(
!
GetVarint64
(
&
raw_val
,
&
u64_val
))
{
log_error
();
continue
;
}
new_table_properties
->
ratio_expire_time
=
u64_val
;
}
else
if
(
key
==
TablePropertiesNames
::
kScanGapExpireTime
)
{
uint64_t
u64_val
;
if
(
!
GetVarint64
(
&
raw_val
,
&
u64_val
))
{
log_error
();
continue
;
}
new_table_properties
->
scan_gap_expire_time
=
u64_val
;
}
else
{
// handle user-collected properties
new_table_properties
->
user_collected_properties
.
insert
(
...
...
table/table_properties.cc
浏览文件 @
a4d77e92
...
...
@@ -181,17 +181,20 @@ void TableProperties::Add(const TableProperties& tp) {
num_deletions
+=
tp
.
num_deletions
;
num_merge_operands
+=
tp
.
num_merge_operands
;
num_range_deletions
+=
tp
.
num_range_deletions
;
uint64_t
max_uint64_t
=
std
::
numeric_limits
<
uint64_t
>::
max
();
if
(
max_uint64_t
-
ratio_expire_time
>
tp
.
ratio_expire_time
)
{
ratio_expire_time
+=
tp
.
ratio_expire_time
;
}
else
{
ratio_expire_time
=
max_uint64_t
;
}
if
(
max_uint64_t
-
scan_gap_expire_time
>
tp
.
scan_gap_expire_time
)
{
scan_gap_expire_time
+=
tp
.
scan_gap_expire_time
;
}
else
{
scan_gap_expire_time
=
max_uint64_t
;
}
ratio_expire_time
=
std
::
min
(
ratio_expire_time
,
tp
.
ratio_expire_time
);
scan_gap_expire_time
=
std
::
min
(
scan_gap_expire_time
,
tp
.
scan_gap_expire_time
);
// uint64_t max_uint64_t = std::numeric_limits<uint64_t>::max();
// if (max_uint64_t - ratio_expire_time > tp.ratio_expire_time) {
// ratio_expire_time += tp.ratio_expire_time;
// } else {
// ratio_expire_time = max_uint64_t;
// }
// if (max_uint64_t - scan_gap_expire_time > tp.scan_gap_expire_time) {
// scan_gap_expire_time += tp.scan_gap_expire_time;
// } else {
// scan_gap_expire_time = max_uint64_t;
// }
}
const
std
::
string
TablePropertiesNames
::
kDataSize
=
"rocksdb.data.size"
;
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录