Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
indiff7643
Terarkdb
提交
465d9354
T
Terarkdb
项目概览
indiff7643
/
Terarkdb
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
Terarkdb
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
465d9354
编写于
12月 22, 2020
作者:
U
ustcwelcome
提交者:
奏之章
2月 08, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
[feature] add parameters tp cf_options and unit test #5
上级
1cfe83a6
变更
13
隐藏空白更改
内联
并排
Showing
13 changed file
with
308 addition
and
112 deletion
+308
-112
CMakeLists.txt
CMakeLists.txt
+0
-63
db/version_edit.cc
db/version_edit.cc
+2
-2
include/rocksdb/options.h
include/rocksdb/options.h
+9
-0
include/rocksdb/table_properties.h
include/rocksdb/table_properties.h
+5
-0
options/cf_options.cc
options/cf_options.cc
+4
-0
options/cf_options.h
options/cf_options.h
+10
-2
options/options.cc
options/options.cc
+3
-0
options/options_helper.cc
options/options_helper.cc
+30
-1
options/options_helper.h
options/options_helper.h
+1
-0
options/options_settable_test.cc
options/options_settable_test.cc
+6
-1
table/block_based_table_builder.cc
table/block_based_table_builder.cc
+51
-23
table/block_based_table_builder.h
table/block_based_table_builder.h
+3
-0
table/block_based_table_builder_test.cc
table/block_based_table_builder_test.cc
+184
-20
未找到文件。
CMakeLists.txt
浏览文件 @
465d9354
...
...
@@ -1006,69 +1006,6 @@ if(WITH_TESTS)
options/options_test.cc
table/block_based_filter_block_test.cc
table/block_based_table_builder_test.cc
table/block_test.cc
table/cleanable_test.cc
table/cuckoo_table_builder_test.cc
table/cuckoo_table_reader_test.cc
table/data_block_hash_index_test.cc
table/full_filter_block_test.cc
table/merger_test.cc
table/sst_file_reader_test.cc
table/table_test.cc
tools/ldb_cmd_test.cc
tools/reduce_levels_test.cc
tools/sst_dump_test.cc
tools/trace_analyzer_test.cc
util/arena_test.cc
util/auto_roll_logger_test.cc
util/autovector_test.cc
util/bloom_test.cc
util/coding_test.cc
util/crc32c_test.cc
util/delete_scheduler_test.cc
util/dynamic_bloom_test.cc
util/event_logger_test.cc
util/file_reader_writer_test.cc
util/filelock_test.cc
util/hash_test.cc
util/heap_test.cc
util/lazy_buffer_test.cc
util/rate_limiter_test.cc
util/repeatable_thread_test.cc
util/slice_transform_test.cc
util/timer_queue_test.cc
util/thread_list_test.cc
util/thread_local_test.cc
utilities/backupable/backupable_db_test.cc
utilities/cassandra/cassandra_functional_test.cc
utilities/cassandra/cassandra_format_test.cc
utilities/cassandra/cassandra_row_merge_test.cc
utilities/cassandra/cassandra_serialize_test.cc
utilities/flink/flink_compaction_filter_test.cc
utilities/checkpoint/checkpoint_test.cc
utilities/column_aware_encoding_test.cc
utilities/date_tiered/date_tiered_test.cc
utilities/document/document_db_test.cc
utilities/document/json_document_test.cc
utilities/geodb/geodb_test.cc
utilities/lua/rocks_lua_test.cc
utilities/memory/memory_test.cc
utilities/merge_operators/string_append/stringappend_test.cc
utilities/object_registry_test.cc
utilities/option_change_migration/option_change_migration_test.cc
utilities/options/options_util_test.cc
utilities/persistent_cache/hash_table_test.cc
utilities/persistent_cache/persistent_cache_test.cc
utilities/redis/redis_lists_test.cc
utilities/spatialdb/spatial_db_test.cc
utilities/simulator_cache/sim_cache_test.cc
utilities/table_properties_collectors/compact_on_deletion_collector_test.cc
utilities/transactions/optimistic_transaction_test.cc
utilities/transactions/transaction_test.cc
utilities/transactions/write_prepared_transaction_test.cc
utilities/transactions/write_unprepared_transaction_test.cc
utilities/ttl/ttl_test.cc
utilities/write_batch_with_index/write_batch_with_index_test.cc
)
if
(
WITH_LIBRADOS
)
...
...
db/version_edit.cc
浏览文件 @
465d9354
...
...
@@ -384,12 +384,12 @@ const char* VersionEdit::DecodeNewFile4From(Slice* input) {
f
.
prop
.
raw_value_size
>
0
)
{
f
.
need_upgrade
=
false
;
}
if
(
f
.
prop
.
num_entries
>
0
)
{
if
(
!
field
.
empty
()
)
{
if
(
!
GetVarint64
(
&
field
,
&
f
.
prop
.
ratio_expire_time
))
{
return
error_msg
;
}
}
if
(
f
.
prop
.
num_entries
>
0
)
{
if
(
!
field
.
empty
()
)
{
if
(
!
GetVarint64
(
&
field
,
&
f
.
prop
.
scan_gap_expire_time
))
{
return
error_msg
;
}
...
...
include/rocksdb/options.h
浏览文件 @
465d9354
...
...
@@ -336,6 +336,15 @@ struct ColumnFamilyOptions : public AdvancedColumnFamilyOptions {
// Default: empty
std
::
vector
<
DbPath
>
cf_paths
;
// The ratio percentage of ttl to mark a SST to be compacted.
// Default: 100.0
double
ttl_garbage_collection_percentage
=
100.0
;
// The scan gap of ttl to mark a SST to be compacted.
// If the value not set, we do not enable
// Default: INT_MAX
int
ttl_scan_gap
=
std
::
numeric_limits
<
int
>::
max
();
// Create ColumnFamilyOptions with default values for all fields
ColumnFamilyOptions
();
// Create ColumnFamilyOptions from Options
...
...
include/rocksdb/table_properties.h
浏览文件 @
465d9354
...
...
@@ -221,6 +221,11 @@ struct TablePropertiesBase {
// "nullptr".
std
::
string
value_meta_extractor_name
;
// The name of the ttl extractor used in this table.
// If no ttl extractor is used, `ttl_extractor_name` will be
// "nullptr".
std
::
string
ttl_extractor_name
;
// The name of the prefix extractor used in this table
// If no prefix extractor is used, `prefix_extractor_name` will be "nullptr".
std
::
string
prefix_extractor_name
;
...
...
options/cf_options.cc
浏览文件 @
465d9354
...
...
@@ -186,6 +186,10 @@ void MutableCFOptions::Dump(Logger* log) const {
max_bytes_for_level_multiplier
);
ROCKS_LOG_INFO
(
log
,
" ttl: %"
PRIu64
,
ttl
);
ROCKS_LOG_INFO
(
log
,
" ttl_garbage_collection_percentage: %f"
,
ttl_garbage_collection_percentage
);
ROCKS_LOG_INFO
(
log
,
" ttl_scan_gap: %f"
,
ttl_scan_gap
);
std
::
string
result
;
char
buf
[
10
];
for
(
const
auto
m
:
max_bytes_for_level_multiplier_additional
)
{
...
...
options/cf_options.h
浏览文件 @
465d9354
...
...
@@ -177,7 +177,10 @@ struct MutableCFOptions {
options
.
max_sequential_skip_in_iterations
),
paranoid_file_checks
(
options
.
paranoid_file_checks
),
report_bg_io_stats
(
options
.
report_bg_io_stats
),
compression
(
options
.
compression
)
{
compression
(
options
.
compression
),
ttl_garbage_collection_percentage
(
options
.
ttl_garbage_collection_percentage
),
ttl_scan_gap
(
options
.
ttl_scan_gap
)
{
RefreshDerivedOptions
(
options
.
num_levels
);
}
...
...
@@ -210,7 +213,9 @@ struct MutableCFOptions {
max_sequential_skip_in_iterations
(
0
),
paranoid_file_checks
(
false
),
report_bg_io_stats
(
false
),
compression
(
Snappy_Supported
()
?
kSnappyCompression
:
kNoCompression
)
{}
compression
(
Snappy_Supported
()
?
kSnappyCompression
:
kNoCompression
),
ttl_garbage_collection_percentage
(
100.0
),
ttl_scan_gap
(
std
::
numeric_limits
<
int
>::
max
())
{}
explicit
MutableCFOptions
(
const
Options
&
options
);
...
...
@@ -276,6 +281,9 @@ struct MutableCFOptions {
// Derived options
// Per-level target file size.
std
::
vector
<
uint64_t
>
max_file_size
;
double
ttl_garbage_collection_percentage
;
int
ttl_scan_gap
;
};
uint64_t
MultiplyCheckOverflow
(
uint64_t
op1
,
double
op2
);
...
...
options/options.cc
浏览文件 @
465d9354
...
...
@@ -124,6 +124,9 @@ void ColumnFamilyOptions::Dump(Logger* log) const {
value_meta_extractor_factory
?
value_meta_extractor_factory
->
Name
()
:
"None"
);
ROCKS_LOG_HEADER
(
log
,
" Options.ttl_extractor_factory: %s"
,
ttl_extractor_factory
?
ttl_extractor_factory
->
Name
()
:
"None"
);
ROCKS_LOG_HEADER
(
log
,
" Options.compaction_filter: %s"
,
compaction_filter
?
compaction_filter
->
Name
()
:
"None"
);
ROCKS_LOG_HEADER
(
...
...
options/options_helper.cc
浏览文件 @
465d9354
...
...
@@ -184,6 +184,9 @@ ColumnFamilyOptions BuildColumnFamilyOptions(
cf_opts
.
max_bytes_for_level_multiplier
=
mutable_cf_options
.
max_bytes_for_level_multiplier
;
cf_opts
.
ttl
=
mutable_cf_options
.
ttl
;
cf_opts
.
ttl_garbage_collection_percentage
=
mutable_cf_options
.
ttl_garbage_collection_percentage
;
cf_opts
.
ttl_scan_gap
=
mutable_cf_options
.
ttl_scan_gap
;
cf_opts
.
max_bytes_for_level_multiplier_additional
.
clear
();
for
(
auto
value
:
...
...
@@ -472,6 +475,19 @@ bool ParseValueExtractorFactory(
return
true
;
}
bool
ParseTtlExtractorFactory
(
const
std
::
string
&
ttl_extractor_factory_name
,
std
::
shared_ptr
<
const
TtlExtractorFactory
>*
ttl_extractor_factory
)
{
if
(
ttl_extractor_factory_name
==
kNullptrString
)
{
ttl_extractor_factory
->
reset
();
return
true
;
}
ttl_extractor_factory
->
reset
(
TtlExtractorFactory
::
create
(
ttl_extractor_factory_name
,
Slice
()
/* ttl_extractor_factory_option */
));
// ttl extractor can be null, so always return true
return
true
;
}
bool
ParseOptionHelper
(
char
*
opt_address
,
const
OptionType
&
opt_type
,
const
std
::
string
&
value
)
{
switch
(
opt_type
)
{
...
...
@@ -586,6 +602,10 @@ bool ParseOptionHelper(char* opt_address, const OptionType& opt_type,
value
,
reinterpret_cast
<
std
::
shared_ptr
<
const
ValueExtractorFactory
>*>
(
opt_address
));
case
OptionType
::
kTtlExtractorFactory
:
return
ParseTtlExtractorFactory
(
value
,
reinterpret_cast
<
std
::
shared_ptr
<
const
TtlExtractorFactory
>*>
(
opt_address
));
default:
return
false
;
}
...
...
@@ -2032,7 +2052,16 @@ std::unordered_map<std::string, OptionTypeInfo>
{
"ttl"
,
{
offset_of
(
&
ColumnFamilyOptions
::
ttl
),
OptionType
::
kUInt64T
,
OptionVerificationType
::
kNormal
,
true
,
offsetof
(
struct
MutableCFOptions
,
ttl
)}}};
offsetof
(
struct
MutableCFOptions
,
ttl
)}},
{
"ttl_garbage_collection_percentage"
,
{
offset_of
(
&
ColumnFamilyOptions
::
ttl_garbage_collection_percentage
),
OptionType
::
kDouble
,
OptionVerificationType
::
kNormal
,
true
,
offsetof
(
struct
MutableCFOptions
,
ttl_garbage_collection_percentage
)}},
{
"ttl_scan_gap"
,
{
offset_of
(
&
ColumnFamilyOptions
::
ttl_scan_gap
),
OptionType
::
kInt
,
OptionVerificationType
::
kNormal
,
true
,
offsetof
(
struct
MutableCFOptions
,
ttl_scan_gap
)}}};
std
::
unordered_map
<
std
::
string
,
OptionTypeInfo
>
OptionsHelper
::
fifo_compaction_options_type_info
=
{
...
...
options/options_helper.h
浏览文件 @
465d9354
...
...
@@ -83,6 +83,7 @@ enum class OptionType {
kEntropyAlgo
,
kWriteBufferFlushPri
,
kValueExtractorFactory
,
kTtlExtractorFactory
,
kUnknown
};
...
...
options/options_settable_test.cc
浏览文件 @
465d9354
...
...
@@ -200,7 +200,8 @@ TEST_F(OptionsSettableTest, DBOptionsAllFieldsSettable) {
{
offsetof
(
struct
DBOptions
,
listeners
),
sizeof
(
std
::
vector
<
std
::
shared_ptr
<
EventListener
>>
)},
{
offsetof
(
struct
DBOptions
,
row_cache
),
sizeof
(
std
::
shared_ptr
<
Cache
>
)},
{
offsetof
(
struct
DBOptions
,
metrics_reporter_factory
),
sizeof
(
std
::
shared_ptr
<
MetricsReporterFactory
>
)},
{
offsetof
(
struct
DBOptions
,
metrics_reporter_factory
),
sizeof
(
std
::
shared_ptr
<
MetricsReporterFactory
>
)},
{
offsetof
(
struct
DBOptions
,
wal_filter
),
sizeof
(
const
WalFilter
*
)},
};
...
...
@@ -353,6 +354,8 @@ TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) {
sizeof
(
std
::
shared_ptr
<
MergeOperator
>
)},
{
offset_of
(
&
ColumnFamilyOptions
::
value_meta_extractor_factory
),
sizeof
(
const
std
::
shared_ptr
<
ValueExtractorFactory
>
)},
{
offset_of
(
&
ColumnFamilyOptions
::
ttl_extractor_factory
),
sizeof
(
const
std
::
shared_ptr
<
TtlExtractorFactory
>
)},
{
offset_of
(
&
ColumnFamilyOptions
::
compaction_filter
),
sizeof
(
const
CompactionFilter
*
)},
{
offset_of
(
&
ColumnFamilyOptions
::
compaction_filter_factory
),
...
...
@@ -465,6 +468,8 @@ TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) {
"blob_gc_ratio=0.05;"
"report_bg_io_stats=true;"
"ttl=60;"
"ttl_garbage_collection_percentage=100.0;"
"ttl_scan_gap=2147483647;"
"compaction_options_fifo={max_table_files_size=3;ttl=100;allow_"
"compaction=false;};"
,
new_options
));
...
...
table/block_based_table_builder.cc
浏览文件 @
465d9354
...
...
@@ -375,6 +375,14 @@ BlockBasedTableBuilder::BlockBasedTableBuilder(
rep_
=
new
Rep
(
builder_options
,
sanitized_table_options
,
column_family_id
,
file
);
if
(
builder_options
.
moptions
.
ttl_garbage_collection_percentage
>=
100.0
&&
builder_options
.
moptions
.
ttl_scan_gap
==
std
::
numeric_limits
<
int
>::
max
())
{
enable_row_ttl_
=
false
;
}
else
{
enable_row_ttl_
=
true
;
}
if
(
rep_
->
filter_builder
!=
nullptr
)
{
rep_
->
filter_builder
->
StartBlock
(
0
);
}
...
...
@@ -384,16 +392,23 @@ BlockBasedTableBuilder::BlockBasedTableBuilder(
&
rep_
->
compressed_cache_key_prefix
[
0
],
&
rep_
->
compressed_cache_key_prefix_size
);
}
if
(
true
)
{
if
(
is_row_ttl_enable
())
{
if
(
builder_options
.
moptions
.
ttl_garbage_collection_percentage
<
100.0
)
{
ttl_histogram_
=
std
::
make_unique
<
HistogramImpl
>
();
}
else
{
ttl_histogram_
.
reset
();
}
TtlExtractorContext
ttl_extractor_context
;
ttl_extractor_context
.
column_family_id
=
column_family_id
;
ttl_histogram_
=
std
::
make_unique
<
HistogramImpl
>
();
ttl_extractor_
=
builder_options
.
ioptions
.
ttl_extractor_factory
->
CreateTtlExtractor
(
ttl_extractor_context
);
ttl_seconds_slice_window_
.
clear
();
slice_index_
=
0
;
min_ttl_seconds_
=
std
::
numeric_limits
<
uint64_t
>::
max
();
}
else
{
ttl_histogram_
.
reset
();
ttl_extractor_
.
reset
();
}
}
...
...
@@ -453,7 +468,7 @@ Status BlockBasedTableBuilder::Add(const Slice& key,
r
->
props
.
num_merge_operands
++
;
}
if
(
true
)
{
if
(
is_row_ttl_enable
()
)
{
EntryType
entry_type
=
GetEntryType
(
value_type
);
if
(
entry_type
==
kEntryMerge
||
entry_type
==
kEntryPut
)
{
bool
has_ttl
=
false
;
...
...
@@ -465,22 +480,26 @@ Status BlockBasedTableBuilder::Add(const Slice& key,
return
s
;
}
if
(
has_ttl
)
{
assert
(
ttl_histogram_
!=
nullptr
);
uint64_t
key_ttl
=
static_cast
<
uint64_t
>
(
ttl
.
count
());
ttl_histogram_
->
Add
(
key_ttl
);
if
(
r
->
moptions
.
ttl_garbage_collection_percentage
<
100.0
)
{
assert
(
ttl_histogram_
!=
nullptr
);
ttl_histogram_
->
Add
(
key_ttl
);
}
// left for scan
int
slice_length
=
10
;
if
(
ttl_seconds_slice_window_
.
size
()
<
slice_length
)
{
ttl_seconds_slice_window_
.
emplace_back
(
key_ttl
);
min_ttl_seconds_
=
std
::
min
(
min_ttl_seconds_
,
key_ttl
);
}
else
{
assert
(
slice_length
==
ttl_seconds_slice_window_
.
size
());
ttl_seconds_slice_window_
[
slice_index_
]
=
key_ttl
;
min_ttl_seconds_
=
std
::
min
(
min_ttl_seconds_
,
*
std
::
max_element
(
ttl_seconds_slice_window_
.
begin
(),
ttl_seconds_slice_window_
.
end
()));
slice_index_
=
(
slice_index_
+
1
)
%
slice_length
;
int
slice_length
=
r
->
moptions
.
ttl_scan_gap
;
if
(
slice_length
<
std
::
numeric_limits
<
int
>::
max
())
{
if
(
ttl_seconds_slice_window_
.
size
()
<
slice_length
)
{
ttl_seconds_slice_window_
.
emplace_back
(
key_ttl
);
min_ttl_seconds_
=
std
::
min
(
min_ttl_seconds_
,
key_ttl
);
}
else
{
assert
(
slice_length
==
ttl_seconds_slice_window_
.
size
());
ttl_seconds_slice_window_
[
slice_index_
]
=
key_ttl
;
min_ttl_seconds_
=
std
::
min
(
min_ttl_seconds_
,
*
std
::
max_element
(
ttl_seconds_slice_window_
.
begin
(),
ttl_seconds_slice_window_
.
end
()));
slice_index_
=
(
slice_index_
+
1
)
%
slice_length
;
}
}
}
}
...
...
@@ -844,6 +863,10 @@ void BlockBasedTableBuilder::WritePropertiesBlock(
rep_
->
ioptions
.
value_meta_extractor_factory
!=
nullptr
?
rep_
->
ioptions
.
value_meta_extractor_factory
->
Name
()
:
"nullptr"
;
rep_
->
props
.
ttl_extractor_name
=
rep_
->
ioptions
.
ttl_extractor_factory
!=
nullptr
?
rep_
->
ioptions
.
ttl_extractor_factory
->
Name
()
:
"nullptr"
;
rep_
->
props
.
compression_name
=
CompressionTypeToString
(
rep_
->
compression_ctx
.
type
());
rep_
->
props
.
prefix_extractor_name
=
...
...
@@ -876,13 +899,18 @@ void BlockBasedTableBuilder::WritePropertiesBlock(
rep_
->
props
.
creation_time
=
rep_
->
creation_time
;
rep_
->
props
.
oldest_key_time
=
rep_
->
oldest_key_time
;
if
(
true
)
{
assert
(
ttl_histogram_
!=
nullptr
);
uint64_t
percentile_ratio_ttl
=
static_cast
<
uint64_t
>
(
ttl_histogram_
->
Percentile
(
10.0
));
if
(
is_row_ttl_enable
())
{
uint64_t
now_seconds
=
rep_
->
ioptions
.
env
->
NowMicros
()
/
1000000ul
;
rep_
->
props
.
ratio_expire_time
=
now_seconds
+
percentile_ratio_ttl
;
rep_
->
props
.
scan_gap_expire_time
=
now_seconds
+
min_ttl_seconds_
;
if
(
rep_
->
moptions
.
ttl_garbage_collection_percentage
<
100.0
)
{
assert
(
ttl_histogram_
!=
nullptr
);
uint64_t
percentile_ratio_ttl
=
static_cast
<
uint64_t
>
(
ttl_histogram_
->
Percentile
(
rep_
->
moptions
.
ttl_garbage_collection_percentage
));
rep_
->
props
.
ratio_expire_time
=
now_seconds
+
percentile_ratio_ttl
;
}
if
(
rep_
->
moptions
.
ttl_scan_gap
<
std
::
numeric_limits
<
int
>::
max
())
{
rep_
->
props
.
scan_gap_expire_time
=
now_seconds
+
min_ttl_seconds_
;
}
ROCKS_LOG_INFO
(
rep_
->
ioptions
.
info_log
,
"[%s] ratio_expire_time:%"
PRIu64
", scan_gap_expire_time:%"
PRIu64
"."
,
...
...
table/block_based_table_builder.h
浏览文件 @
465d9354
...
...
@@ -82,6 +82,8 @@ class BlockBasedTableBuilder : public TableBuilder {
bool
NeedCompact
()
const
override
;
bool
is_row_ttl_enable
()
const
{
return
enable_row_ttl_
;
}
// Get table properties
TableProperties
GetTableProperties
()
const
override
;
...
...
@@ -133,6 +135,7 @@ class BlockBasedTableBuilder : public TableBuilder {
std
::
vector
<
uint64_t
>
ttl_seconds_slice_window_
;
uint64_t
min_ttl_seconds_
;
int
slice_index_
;
bool
enable_row_ttl_
;
};
Slice
CompressBlock
(
const
Slice
&
raw
,
const
CompressionContext
&
compression_ctx
,
...
...
table/block_based_table_builder_test.cc
浏览文件 @
465d9354
...
...
@@ -28,6 +28,47 @@ class TestTtlExtractor : public TtlExtractor {
}
};
class
TestEnv
:
public
EnvWrapper
{
public:
explicit
TestEnv
()
:
EnvWrapper
(
Env
::
Default
()),
close_count
(
0
)
{}
class
TestLogger
:
public
Logger
{
public:
using
Logger
::
Logv
;
TestLogger
(
TestEnv
*
env_ptr
)
:
Logger
()
{
env
=
env_ptr
;
}
~
TestLogger
()
{
if
(
!
closed_
)
{
CloseHelper
();
}
}
virtual
void
Logv
(
const
char
*
/*format*/
,
va_list
/*ap*/
)
override
{};
protected:
virtual
Status
CloseImpl
()
override
{
return
CloseHelper
();
}
private:
Status
CloseHelper
()
{
env
->
CloseCountInc
();
;
return
Status
::
IOError
();
}
TestEnv
*
env
;
};
void
CloseCountInc
()
{
close_count
++
;
}
int
GetCloseCount
()
{
return
close_count
;
}
virtual
Status
NewLogger
(
const
std
::
string
&
/*fname*/
,
std
::
shared_ptr
<
Logger
>*
result
)
{
result
->
reset
(
new
TestLogger
(
this
));
return
Status
::
OK
();
}
private:
int
close_count
;
};
class
TestTtlExtractorFactory
:
public
TtlExtractorFactory
{
using
TtlContext
=
TtlExtractorContext
;
...
...
@@ -47,7 +88,7 @@ class BlockBasedTableBuilderTest : public testing::Test {
//
};
TEST_F
(
BlockBasedTableBuilderTest
,
SimpleTest
)
{
TEST_F
(
BlockBasedTableBuilderTest
,
SimpleTest
1
)
{
BlockBasedTableOptions
blockbasedtableoptions
;
BlockBasedTableFactory
factory
(
blockbasedtableoptions
);
...
...
@@ -55,6 +96,33 @@ TEST_F(BlockBasedTableBuilderTest, SimpleTest) {
std
::
unique_ptr
<
WritableFileWriter
>
file_writer
(
test
::
GetWritableFileWriter
(
new
test
::
StringSink
(),
""
/* don't care */
));
Options
options
;
std
::
string
dbname
=
test
::
PerThreadDBPath
(
"block_based_table_builder_test"
);
ASSERT_OK
(
DestroyDB
(
dbname
,
options
));
DB
*
db
=
nullptr
;
TestEnv
*
env
=
new
TestEnv
();
options
.
info_log
.
reset
(
new
TestEnv
::
TestLogger
(
env
));
options
.
create_if_missing
=
true
;
options
.
env
=
env
;
Status
s
=
DB
::
Open
(
options
,
dbname
,
&
db
);
ASSERT_OK
(
s
);
ASSERT_TRUE
(
db
!=
nullptr
);
s
=
db
->
Close
();
delete
db
;
// Env* env = Env::Default();
// options.db_log_dir = std::string("log");
// options.info_log = nullptr;
// env->CreateDirIfMissing(options.db_log_dir);
// if (options.info_log == nullptr) {
// Status s = CreateLoggerFromOptions(".", options, &options.info_log);
// if (!s.ok()) {
// // No place suitable for logging
// options.info_log = nullptr;
// }
// }
const
ImmutableCFOptions
ioptions
(
options
);
const
MutableCFOptions
moptions
(
options
);
InternalKeyComparator
ikc
(
options
.
comparator
);
...
...
@@ -82,26 +150,122 @@ TEST_F(BlockBasedTableBuilderTest, SimpleTest) {
ASSERT_OK
(
builder
->
Finish
(
nullptr
,
nullptr
));
file_writer
->
Flush
();
// test::StringSink* ss =
// static_cast<test::StringSink*>(file_writer->writable_file());
// std::unique_ptr<RandomAccessFileReader> file_reader(
// test::GetRandomAccessFileReader(
// new test::StringSource(ss->contents(), 72242, true)));
// TableProperties* props = nullptr;
// auto s = ReadTableProperties(file_reader.get(), ss->contents().size(),
// kBlockBasedTableMagicNumber, ioptions, &props,
// true /* compression_type_missing */);
// std::unique_ptr<TableProperties> props_guard(props);
// ASSERT_OK(s);
// ASSERT_EQ(0ul, props->index_size);
// ASSERT_EQ(0ul, props->filter_size);
// ASSERT_EQ(16ul * 26, props->raw_key_size);
// ASSERT_EQ(28ul * 26, props->raw_value_size);
// ASSERT_EQ(26ul, props->num_entries);
// ASSERT_EQ(1ul, props->num_data_blocks);
test
::
StringSink
*
ss
=
static_cast
<
test
::
StringSink
*>
(
file_writer
->
writable_file
());
std
::
unique_ptr
<
RandomAccessFileReader
>
file_reader
(
test
::
GetRandomAccessFileReader
(
new
test
::
StringSource
(
ss
->
contents
(),
72242
,
true
)));
TableProperties
*
props
=
nullptr
;
s
=
ReadTableProperties
(
file_reader
.
get
(),
ss
->
contents
().
size
(),
kBlockBasedTableMagicNumber
,
ioptions
,
&
props
,
true
/* compression_type_missing */
);
std
::
unique_ptr
<
TableProperties
>
props_guard
(
props
);
ASSERT_OK
(
s
);
ASSERT_EQ
(
0ul
,
props
->
filter_size
);
ASSERT_EQ
(
16ul
*
26
,
props
->
raw_key_size
);
ASSERT_EQ
(
28ul
*
26
,
props
->
raw_value_size
);
ASSERT_EQ
(
26ul
,
props
->
num_entries
);
ASSERT_EQ
(
1ul
,
props
->
num_data_blocks
);
ASSERT_EQ
(
std
::
numeric_limits
<
uint64_t
>::
max
(),
props
->
ratio_expire_time
);
ASSERT_EQ
(
std
::
numeric_limits
<
uint64_t
>::
max
(),
props
->
scan_gap_expire_time
);
// delete env;
// env = nullptr;
delete
options
.
env
;
}
TEST_F
(
BlockBasedTableBuilderTest
,
SimpleTest2
)
{
BlockBasedTableOptions
blockbasedtableoptions
;
BlockBasedTableFactory
factory
(
blockbasedtableoptions
);
test
::
StringSink
sink
;
std
::
unique_ptr
<
WritableFileWriter
>
file_writer
(
test
::
GetWritableFileWriter
(
new
test
::
StringSink
(),
""
/* don't care */
));
Options
options
;
std
::
string
dbname
=
test
::
PerThreadDBPath
(
"block_based_table_builder_test"
);
ASSERT_OK
(
DestroyDB
(
dbname
,
options
));
DB
*
db
=
nullptr
;
TestEnv
*
env
=
new
TestEnv
();
options
.
info_log
.
reset
(
new
TestEnv
::
TestLogger
(
env
));
options
.
create_if_missing
=
true
;
options
.
env
=
env
;
options
.
ttl_garbage_collection_percentage
=
100.0
;
options
.
ttl_scan_gap
=
10
;
options
.
ttl_extractor_factory
.
reset
(
new
TestTtlExtractorFactory
());
Status
s
=
DB
::
Open
(
options
,
dbname
,
&
db
);
ASSERT_OK
(
s
);
ASSERT_TRUE
(
db
!=
nullptr
);
s
=
db
->
Close
();
delete
db
;
// Env* env = Env::Default();
// options.db_log_dir = std::string("log");
// options.info_log = nullptr;
// env->CreateDirIfMissing(options.db_log_dir);
// if (options.info_log == nullptr) {
// Status s = CreateLoggerFromOptions(".", options, &options.info_log);
// if (!s.ok()) {
// // No place suitable for logging
// options.info_log = nullptr;
// }
// }
const
ImmutableCFOptions
ioptions
(
options
);
const
MutableCFOptions
moptions
(
options
);
InternalKeyComparator
ikc
(
options
.
comparator
);
std
::
vector
<
std
::
unique_ptr
<
IntTblPropCollectorFactory
>>
int_tbl_prop_collector_factories
;
// int_tbl_prop_collector_factories.emplace_back(
// make_unique<BlockBasedTableBuilder::BlockBasedTablePropertiesCollector>(
// BlockBasedTableOptions::IndexType::kBinarySearch, false, false));
std
::
string
column_family_name
;
int
unknown_level
=
-
1
;
std
::
unique_ptr
<
TableBuilder
>
builder
(
factory
.
NewTableBuilder
(
TableBuilderOptions
(
ioptions
,
moptions
,
ikc
,
&
int_tbl_prop_collector_factories
,
kNoCompression
,
CompressionOptions
(),
nullptr
/* compression_dict */
,
false
/* skip_filters */
,
column_family_name
,
unknown_level
,
0
/* compaction_load */
),
TablePropertiesCollectorFactory
::
Context
::
kUnknownColumnFamily
,
file_writer
.
get
()));
for
(
char
c
=
'a'
;
c
<=
'z'
;
++
c
)
{
std
::
string
key
(
8
,
c
);
key
.
append
(
"
\1
"
);
// PlainTable expects internal key structure
std
::
string
value
(
28
,
c
+
42
);
ASSERT_OK
(
builder
->
Add
(
key
,
LazyBuffer
(
value
)));
}
ASSERT_OK
(
builder
->
Finish
(
nullptr
,
nullptr
));
file_writer
->
Flush
();
test
::
StringSink
*
ss
=
static_cast
<
test
::
StringSink
*>
(
file_writer
->
writable_file
());
std
::
unique_ptr
<
RandomAccessFileReader
>
file_reader
(
test
::
GetRandomAccessFileReader
(
new
test
::
StringSource
(
ss
->
contents
(),
72242
,
true
)));
TableProperties
*
props
=
nullptr
;
s
=
ReadTableProperties
(
file_reader
.
get
(),
ss
->
contents
().
size
(),
kBlockBasedTableMagicNumber
,
ioptions
,
&
props
,
true
/* compression_type_missing */
);
std
::
unique_ptr
<
TableProperties
>
props_guard
(
props
);
ASSERT_OK
(
s
);
ASSERT_EQ
(
0ul
,
props
->
filter_size
);
ASSERT_EQ
(
16ul
*
26
,
props
->
raw_key_size
);
ASSERT_EQ
(
28ul
*
26
,
props
->
raw_value_size
);
ASSERT_EQ
(
26ul
,
props
->
num_entries
);
ASSERT_EQ
(
1ul
,
props
->
num_data_blocks
);
ASSERT_EQ
(
std
::
numeric_limits
<
uint64_t
>::
max
(),
props
->
ratio_expire_time
);
ASSERT_NE
(
std
::
numeric_limits
<
uint64_t
>::
max
(),
props
->
scan_gap_expire_time
);
// delete env;
// env = nullptr;
delete
options
.
env
;
}
}
// namespace rocksdb
int
main
(
int
argc
,
char
**
argv
)
{
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录