Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
indiff7643
Terarkdb
提交
66bc2cc7
T
Terarkdb
项目概览
indiff7643
/
Terarkdb
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
Terarkdb
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
66bc2cc7
编写于
12月 23, 2020
作者:
U
ustcwelcome
提交者:
奏之章
2月 08, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
[feature] bug-fix of ttl and unit test #5
上级
742d0528
变更
5
隐藏空白更改
内联
并排
Showing
5 changed file
with
46 addition
and
26 deletion
+46
-26
include/rocksdb/table_properties.h
include/rocksdb/table_properties.h
+2
-0
table/block_based_table_builder.cc
table/block_based_table_builder.cc
+30
-14
table/block_based_table_builder.h
table/block_based_table_builder.h
+1
-0
table/block_based_table_builder_ttl_test.cc
table/block_based_table_builder_ttl_test.cc
+11
-11
util/testutil.h
util/testutil.h
+2
-1
未找到文件。
include/rocksdb/table_properties.h
浏览文件 @
66bc2cc7
...
...
@@ -187,6 +187,8 @@ struct TablePropertiesBase {
uint64_t
num_merge_operands
=
0
;
// the number of range deletions in this table
uint64_t
num_range_deletions
=
0
;
// the number of k-v pairs has raw ttl in this table
// uint64_t num_has_raw_ttl = 0;
// format version, reserved for backward compatibility
uint64_t
format_version
=
0
;
// If 0, key is variable length. Otherwise number of bytes for each key.
...
...
table/block_based_table_builder.cc
浏览文件 @
66bc2cc7
...
...
@@ -398,8 +398,10 @@ BlockBasedTableBuilder::BlockBasedTableBuilder(
}
else
{
ttl_histogram_
.
reset
();
}
num_has_row_ttl_
=
0
;
TtlExtractorContext
ttl_extractor_context
;
ttl_extractor_context
.
column_family_id
=
column_family_id
;
assert
(
builder_options
.
ioptions
.
ttl_extractor_factory
!=
nullptr
);
ttl_extractor_
=
builder_options
.
ioptions
.
ttl_extractor_factory
->
CreateTtlExtractor
(
ttl_extractor_context
);
...
...
@@ -479,28 +481,29 @@ Status BlockBasedTableBuilder::Add(const Slice& key,
if
(
!
s
.
ok
())
{
return
s
;
}
uint64_t
key_ttl
=
std
::
numeric_limits
<
uint64_t
>::
max
();
if
(
has_ttl
)
{
num_has_row_ttl_
++
;
uint64_t
key_ttl
=
std
::
numeric_limits
<
uint64_t
>::
max
();
key_ttl
=
static_cast
<
uint64_t
>
(
ttl
.
count
());
}
if
(
r
->
moptions
.
ttl_garbage_collection_percentage
<
100.0
)
{
assert
(
ttl_histogram_
!=
nullptr
);
ttl_histogram_
->
Add
(
key_ttl
);
}
if
(
has_ttl
)
{
if
(
r
->
moptions
.
ttl_garbage_collection_percentage
<
100.0
)
{
assert
(
ttl_histogram_
!=
nullptr
);
ttl_histogram_
->
Add
(
key_ttl
);
}
int
slice_length
=
r
->
moptions
.
ttl_scan_gap
;
if
(
slice_length
<
std
::
numeric_limits
<
int
>::
max
())
{
if
(
ttl_seconds_slice_window_
.
size
()
<
slice_length
)
{
ttl_seconds_slice_window_
.
emplace_back
(
key_ttl
);
min_ttl_seconds_
=
std
::
min
(
min_ttl_seconds_
,
key_ttl
);
}
else
{
assert
(
slice_length
==
ttl_seconds_slice_window_
.
size
());
ttl_seconds_slice_window_
[
slice_index_
]
=
key_ttl
;
slice_index_
=
(
slice_index_
+
1
)
%
slice_length
;
}
if
(
ttl_seconds_slice_window_
.
size
()
==
slice_length
)
{
min_ttl_seconds_
=
std
::
min
(
min_ttl_seconds_
,
*
std
::
max_element
(
ttl_seconds_slice_window_
.
begin
(),
ttl_seconds_slice_window_
.
end
()));
slice_index_
=
(
slice_index_
+
1
)
%
slice_length
;
}
}
}
else
{
...
...
@@ -906,15 +909,28 @@ void BlockBasedTableBuilder::WritePropertiesBlock(
if
(
is_row_ttl_enable
())
{
uint64_t
now_seconds
=
rep_
->
ioptions
.
env
->
NowMicros
()
/
1000000ul
;
uint64_t
max_uint64_t
=
std
::
numeric_limits
<
uint64_t
>::
max
();
if
(
rep_
->
moptions
.
ttl_garbage_collection_percentage
<
100.0
)
{
assert
(
ttl_histogram_
!=
nullptr
);
uint64_t
percentile_ratio_ttl
=
static_cast
<
uint64_t
>
(
ttl_histogram_
->
Percentile
(
rep_
->
moptions
.
ttl_garbage_collection_percentage
));
rep_
->
props
.
ratio_expire_time
=
now_seconds
+
percentile_ratio_ttl
;
uint64_t
percentile_ratio_ttl
=
max_uint64_t
;
if
(
!
ttl_histogram_
->
Empty
())
{
percentile_ratio_ttl
=
static_cast
<
uint64_t
>
(
ttl_histogram_
->
Percentile
(
rep_
->
moptions
.
ttl_garbage_collection_percentage
*
num_has_row_ttl_
/
rep_
->
props
.
num_entries
));
}
if
(
max_uint64_t
-
percentile_ratio_ttl
>
now_seconds
)
{
rep_
->
props
.
ratio_expire_time
=
now_seconds
+
percentile_ratio_ttl
;
}
else
{
rep_
->
props
.
ratio_expire_time
=
max_uint64_t
;
}
}
if
(
rep_
->
moptions
.
ttl_scan_gap
<
std
::
numeric_limits
<
int
>::
max
())
{
rep_
->
props
.
scan_gap_expire_time
=
now_seconds
+
min_ttl_seconds_
;
if
(
max_uint64_t
-
min_ttl_seconds_
>
now_seconds
)
{
rep_
->
props
.
scan_gap_expire_time
=
now_seconds
+
min_ttl_seconds_
;
}
else
{
rep_
->
props
.
scan_gap_expire_time
=
max_uint64_t
;
}
}
ROCKS_LOG_INFO
(
rep_
->
ioptions
.
info_log
,
"[%s] ratio_expire_time:%"
PRIu64
...
...
table/block_based_table_builder.h
浏览文件 @
66bc2cc7
...
...
@@ -136,6 +136,7 @@ class BlockBasedTableBuilder : public TableBuilder {
uint64_t
min_ttl_seconds_
;
int
slice_index_
;
bool
enable_row_ttl_
;
uint64_t
num_has_row_ttl_
;
};
Slice
CompressBlock
(
const
Slice
&
raw
,
const
CompressionContext
&
compression_ctx
,
...
...
table/block_based_table_builder_ttl_test.cc
浏览文件 @
66bc2cc7
...
...
@@ -3,6 +3,7 @@
// #include "include/rocksdb/ttl_extractor.h"
#include "table/block_based_table_builder.h"
#include "util/string_util.h"
#include "util/testharness.h"
#include "util/testutil.h"
...
...
@@ -150,19 +151,17 @@ TEST_F(BlockBasedTableBuilderTest, SimpleTest2) {
std
::
string
dbname
=
test
::
PerThreadDBPath
(
"block_based_table_builder_test"
);
ASSERT_OK
(
DestroyDB
(
dbname
,
options
));
DB
*
db
=
nullptr
;
TestEnv
*
env
=
new
TestEnv
();
options
.
info_log
.
reset
(
new
TestEnv
::
TestLogger
(
env
));
options
.
create_if_missing
=
true
;
options
.
env
=
env
;
options
.
ttl_garbage_collection_percentage
=
50.0
;
options
.
ttl_scan_gap
=
1
0
;
options
.
ttl_garbage_collection_percentage
=
99.9999
;
options
.
ttl_scan_gap
=
1
;
options
.
ttl_extractor_factory
.
reset
(
new
test
::
TestTtlExtractorFactory
());
Status
s
=
DB
::
Open
(
options
,
dbname
,
&
db
);
ASSERT_OK
(
s
);
ASSERT_TRUE
(
db
!=
nullptr
);
s
=
db
->
Close
();
delete
db
;
...
...
@@ -179,13 +178,9 @@ TEST_F(BlockBasedTableBuilderTest, SimpleTest2) {
// }
const
ImmutableCFOptions
ioptions
(
options
);
const
MutableCFOptions
moptions
(
options
);
InternalKeyComparator
ikc
(
options
.
comparator
);
std
::
vector
<
std
::
unique_ptr
<
IntTblPropCollectorFactory
>>
int_tbl_prop_collector_factories
;
// int_tbl_prop_collector_factories.emplace_back(
// make_unique<BlockBasedTableBuilder::BlockBasedTablePropertiesCollector>(
// BlockBasedTableOptions::IndexType::kBinarySearch, false, false));
std
::
string
column_family_name
;
int
unknown_level
=
-
1
;
std
::
unique_ptr
<
TableBuilder
>
builder
(
factory
.
NewTableBuilder
(
...
...
@@ -200,6 +195,11 @@ TEST_F(BlockBasedTableBuilderTest, SimpleTest2) {
std
::
string
key
(
8
,
c
);
key
.
append
(
"
\1
"
);
// PlainTable expects internal key structure
std
::
string
value
(
28
,
c
+
42
);
char
ts_string
[
sizeof
(
uint64_t
)];
uint64_t
ttl
=
100
;
EncodeFixed64
(
ts_string
,
(
uint64_t
)
ttl
);
// AppendNumberTo(&value, ttl);
value
.
append
(
ts_string
,
sizeof
(
uint64_t
));
ASSERT_OK
(
builder
->
Add
(
key
,
LazyBuffer
(
value
)));
}
ASSERT_OK
(
builder
->
Finish
(
nullptr
,
nullptr
));
...
...
@@ -220,15 +220,15 @@ TEST_F(BlockBasedTableBuilderTest, SimpleTest2) {
ASSERT_EQ
(
0ul
,
props
->
filter_size
);
ASSERT_EQ
(
16ul
*
26
,
props
->
raw_key_size
);
ASSERT_EQ
(
28ul
*
26
,
props
->
raw_value_size
);
//
ASSERT_EQ(28ul * 26, props->raw_value_size);
ASSERT_EQ
(
26ul
,
props
->
num_entries
);
ASSERT_EQ
(
1ul
,
props
->
num_data_blocks
);
env
->
SleepForMicroseconds
(
2000000
);
std
::
cout
<<
props
->
ratio_expire_time
<<
std
::
endl
;
std
::
cout
<<
props
->
scan_gap_expire_time
<<
std
::
endl
;
std
::
cout
<<
env
->
NowMicros
()
/
1000000ul
<<
std
::
endl
;
// delete env
;
// env = nullptr;
std
::
cout
<<
std
::
numeric_limits
<
uint64_t
>::
max
()
<<
std
::
endl
;
delete
options
.
env
;
}
...
...
util/testutil.h
浏览文件 @
66bc2cc7
...
...
@@ -820,8 +820,9 @@ class TestTtlExtractor : public TtlExtractor {
assert
(
value_or_meta
.
size
()
>
kTtlLength
);
*
ttl
=
static_cast
<
std
::
chrono
::
seconds
>
(
DecodeFixed64
(
value_or_meta
.
data
()
+
value_or_meta
.
size
()
-
kTtlLength
));
}
else
{
*
has_ttl
=
false
;
}
*
has_ttl
=
false
;
return
Status
::
OK
();
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录