Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
indiff7643
Terarkdb
提交
85df8fbb
T
Terarkdb
项目概览
indiff7643
/
Terarkdb
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
Terarkdb
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
85df8fbb
编写于
6月 17, 2019
作者:
Z
Zhongyi Xie
提交者:
奏之章
2月 08, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
[cherry-pick]
上级
63456d87
变更
24
展开全部
隐藏空白更改
内联
并排
Showing
24 changed file
with
1061 addition
and
38 deletion
+1061
-38
CMakeLists.txt
CMakeLists.txt
+3
-1
Makefile
Makefile
+157
-0
TARGETS
TARGETS
+7
-1
db/db_impl.cc
db/db_impl.cc
+65
-17
db/db_impl.h
db/db_impl.h
+44
-2
db/db_impl_debug.cc
db/db_impl_debug.cc
+2
-2
db/db_impl_open.cc
db/db_impl_open.cc
+117
-1
db/db_options_test.cc
db/db_options_test.cc
+2
-0
db/version_set.cc
db/version_set.cc
+17
-2
include/rocksdb/db.h
include/rocksdb/db.h
+6
-4
include/rocksdb/options.h
include/rocksdb/options.h
+12
-0
include/rocksdb/stats_history.h
include/rocksdb/stats_history.h
+3
-1
monitoring/in_memory_stats_history.cc
monitoring/in_memory_stats_history.cc
+1
-1
monitoring/in_memory_stats_history.h
monitoring/in_memory_stats_history.h
+2
-0
monitoring/persistent_stats_history.cc
monitoring/persistent_stats_history.cc
+171
-0
monitoring/persistent_stats_history.h
monitoring/persistent_stats_history.h
+83
-0
monitoring/stats_history_test.cc
monitoring/stats_history_test.cc
+295
-2
options/db_options.cc
options/db_options.cc
+4
-1
options/db_options.h
options/db_options.h
+1
-0
options/options_helper.cc
options/options_helper.cc
+5
-0
options/options_settable_test.cc
options/options_settable_test.cc
+1
-0
options/options_test.cc
options/options_test.cc
+2
-0
src.mk
src.mk
+58
-3
tools/db_bench_tool.cc
tools/db_bench_tool.cc
+3
-0
未找到文件。
CMakeLists.txt
浏览文件 @
85df8fbb
...
...
@@ -558,7 +558,6 @@ set(SOURCES
db/flush_scheduler.cc
db/forward_iterator.cc
db/internal_stats.cc
db/in_memory_stats_history.cc
db/logs_with_prep_tracker.cc
db/log_reader.cc
db/log_writer.cc
...
...
@@ -599,10 +598,12 @@ set(SOURCES
memtable/write_buffer_manager.cc
monitoring/histogram.cc
monitoring/histogram_windowing.cc
monitoring/in_memory_stats_history.cc
monitoring/instrumented_mutex.cc
monitoring/iostats_context.cc
monitoring/perf_context.cc
monitoring/perf_level.cc
monitoring/persistent_stats_history.cc
monitoring/statistics.cc
monitoring/stats_dump_scheduler.cc
monitoring/thread_status_impl.cc
...
...
@@ -1005,6 +1006,7 @@ if(WITH_TESTS)
monitoring/histogram_test.cc
monitoring/iostats_context_test.cc
monitoring/statistics_test.cc
monitoring/stats_history_test.cc
options/options_settable_test.cc
options/options_test.cc
table/block_based_filter_block_test.cc
...
...
Makefile
浏览文件 @
85df8fbb
...
...
@@ -518,10 +518,160 @@ ALL_SOURCES += $(TEST_MAIN_SOURCES) $(TOOL_MAIN_SOURCES) $(BENCH_MAIN_SOURCES)
TESTS
=
$(
patsubst
%.cc, %,
$(
notdir
$(TEST_MAIN_SOURCES)
))
TESTS
+=
$(
patsubst
%.c, %,
$(
notdir
$(TEST_MAIN_SOURCES_C)
))
<<<<<<<
HEAD
ifeq
($(USE_FOLLY_DISTRIBUTED_MUTEX),1)
TESTS
+=
folly_synchronization_distributed_mutex_test
ALL_SOURCES
+=
third-party/folly/folly/synchronization/test/DistributedMutexTest.cc
endif
=======
TESTS
=
\
db_basic_test
\
db_encryption_test
\
db_test2
\
external_sst_file_basic_test
\
auto_roll_logger_test
\
bloom_test
\
dynamic_bloom_test
\
c_test
\
checkpoint_test
\
crc32c_test
\
coding_test
\
inlineskiplist_test
\
env_basic_test
\
env_test
\
hash_test
\
thread_local_test
\
rate_limiter_test
\
perf_context_test
\
iostats_context_test
\
db_wal_test
\
db_block_cache_test
\
db_test
\
db_blob_index_test
\
db_bloom_filter_test
\
db_iter_test
\
db_iter_stress_test
\
db_log_iter_test
\
db_compaction_filter_test
\
db_compaction_test
\
db_dynamic_level_test
\
db_flush_test
\
db_inplace_update_test
\
db_iterator_test
\
db_memtable_test
\
db_merge_operator_test
\
db_options_test
\
db_range_del_test
\
db_secondary_test
\
db_sst_test
\
db_tailing_iter_test
\
db_io_failure_test
\
db_properties_test
\
db_table_properties_test
\
db_statistics_test
\
db_write_test
\
error_handler_test
\
autovector_test
\
blob_db_test
\
cleanable_test
\
column_family_test
\
table_properties_collector_test
\
arena_test
\
block_test
\
data_block_hash_index_test
\
cache_test
\
corruption_test
\
slice_transform_test
\
dbformat_test
\
fault_injection_test
\
filelock_test
\
filename_test
\
file_reader_writer_test
\
block_based_filter_block_test
\
full_filter_block_test
\
partitioned_filter_block_test
\
hash_table_test
\
histogram_test
\
log_test
\
manual_compaction_test
\
mock_env_test
\
memtable_list_test
\
merge_helper_test
\
memory_test
\
merge_test
\
merger_test
\
util_merge_operators_test
\
options_file_test
\
reduce_levels_test
\
plain_table_db_test
\
comparator_db_test
\
external_sst_file_test
\
prefix_test
\
skiplist_test
\
write_buffer_manager_test
\
stringappend_test
\
cassandra_format_test
\
cassandra_functional_test
\
cassandra_row_merge_test
\
cassandra_serialize_test
\
ttl_test
\
backupable_db_test
\
sim_cache_test
\
version_edit_test
\
version_set_test
\
compaction_picker_test
\
version_builder_test
\
file_indexer_test
\
write_batch_test
\
write_batch_with_index_test
\
write_controller_test
\
deletefile_test
\
obsolete_files_test
\
table_test
\
delete_scheduler_test
\
options_test
\
options_settable_test
\
options_util_test
\
event_logger_test
\
timer_queue_test
\
cuckoo_table_builder_test
\
cuckoo_table_reader_test
\
cuckoo_table_db_test
\
flush_job_test
\
wal_manager_test
\
listener_test
\
compaction_iterator_test
\
compaction_job_test
\
thread_list_test
\
sst_dump_test
\
compact_files_test
\
optimistic_transaction_test
\
write_callback_test
\
heap_test
\
compact_on_deletion_collector_test
\
compaction_job_stats_test
\
option_change_migration_test
\
transaction_test
\
ldb_cmd_test
\
persistent_cache_test
\
statistics_test
\
stats_history_test
\
lru_cache_test
\
object_registry_test
\
repair_test
\
env_timed_test
\
write_prepared_transaction_test
\
write_unprepared_transaction_test
\
db_universal_compaction_test
\
trace_analyzer_test
\
repeatable_thread_test
\
range_tombstone_fragmenter_test
\
range_del_aggregator_test
\
sst_file_reader_test
\
db_secondary_test
\
block_cache_tracer_test
\
block_cache_trace_analyzer_test
\
>>>>>>>
671d15cbd... Persistent Stats: persist stats
history
to disk
(
#5046
)
PARALLEL_TEST
=
\
backupable_db_test
\
...
...
@@ -1781,7 +1931,14 @@ blob_db_test: $(OBJ_DIR)/utilities/blob_db/blob_db_test.o $(TEST_LIBRARY) $(LIBR
repeatable_thread_test
:
$(OBJ_DIR)/util/repeatable_thread_test.o $(TEST_LIBRARY) $(LIBRARY)
$(AM_LINK)
<<<<<<<
HEAD
range_tombstone_fragmenter_test
:
$(OBJ_DIR)/db/range_tombstone_fragmenter_test.o $(TEST_LIBRARY) $(LIBRARY)
=======
stats_history_test
:
monitoring/stats_history_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
$(AM_LINK)
lru_cache_test
:
cache/lru_cache_test.o $(LIBOBJECTS) $(TESTHARNESS)
>>>>>>> 671d15cbd... Persistent Stats
:
persist stats history to disk (
#
5046)
$(AM_LINK)
sst_file_reader_test
:
$(OBJ_DIR)/table/sst_file_reader_test.o $(TEST_LIBRARY) $(LIBRARY)
...
...
TARGETS
浏览文件 @
85df8fbb
...
...
@@ -113,7 +113,6 @@ cpp_library(
"db/flush_job.cc",
"db/flush_scheduler.cc",
"db/forward_iterator.cc",
"db/in_memory_stats_history.cc",
"db/internal_stats.cc",
"db/log_reader.cc",
"db/log_writer.cc",
...
...
@@ -154,10 +153,12 @@ cpp_library(
"memtable/write_buffer_manager.cc",
"monitoring/histogram.cc",
"monitoring/histogram_windowing.cc",
"monitoring/in_memory_stats_history.cc",
"monitoring/instrumented_mutex.cc",
"monitoring/iostats_context.cc",
"monitoring/perf_context.cc",
"monitoring/perf_level.cc",
"monitoring/persistent_stats_history.cc",
"monitoring/statistics.cc",
"monitoring/stats_dump_scheduler.cc",
"monitoring/thread_status_impl.cc",
...
...
@@ -1000,6 +1001,11 @@ ROCKS_TESTS = [
[],
[],
],
[
"stats_history_test",
"monitoring/stats_history_test.cc",
"serial",
],
[
"stringappend_test",
"utilities/merge_operators/string_append/stringappend_test.cc",
...
...
db/db_impl.cc
浏览文件 @
85df8fbb
...
...
@@ -37,7 +37,6 @@
#include "db/external_sst_file_ingestion_job.h"
#include "db/flush_job.h"
#include "db/forward_iterator.h"
#include "db/in_memory_stats_history.h"
#include "db/job_context.h"
#include "db/log_reader.h"
#include "db/log_writer.h"
...
...
@@ -56,10 +55,12 @@
#include "db/write_callback.h"
#include "memtable/hash_linklist_rep.h"
#include "memtable/hash_skiplist_rep.h"
#include "monitoring/in_memory_stats_history.h"
#include "monitoring/iostats_context_imp.h"
#include "monitoring/perf_context_imp.h"
#include "trace_replay/block_cache_tracer.h"
#include "monitoring/stats_dump_scheduler.h"
#include "monitoring/persistent_stats_history.h"
#include "monitoring/thread_status_updater.h"
#include "monitoring/thread_status_util.h"
#include "options/cf_options.h"
...
...
@@ -215,6 +216,7 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname,
log_dir_synced_
(
false
),
log_empty_
(
true
),
default_cf_handle_
(
nullptr
),
persist_stats_cf_handle_
(
nullptr
),
log_sync_cv_
(
&
mutex_
),
total_log_size_
(
0
),
max_total_in_memory_state_
(
0
),
...
...
@@ -625,10 +627,17 @@ Status DBImpl::CloseHelper() {
}
memtable_info_queue_
.
clear
();
if
(
default_cf_handle_
!=
nullptr
)
{
if
(
default_cf_handle_
!=
nullptr
||
persist_stats_cf_handle_
!=
nullptr
)
{
// we need to delete handle outside of lock because it does its own locking
mutex_
.
Unlock
();
delete
default_cf_handle_
;
if
(
default_cf_handle_
)
{
delete
default_cf_handle_
;
default_cf_handle_
=
nullptr
;
}
if
(
persist_stats_cf_handle_
)
{
delete
persist_stats_cf_handle_
;
persist_stats_cf_handle_
=
nullptr
;
}
mutex_
.
Lock
();
}
...
...
@@ -780,7 +789,7 @@ void DBImpl::StartStatsDumpScheduler() {
}
// esitmate the total size of stats_history_
size_t
DBImpl
::
Esti
amte
StatsHistorySize
()
const
{
size_t
DBImpl
::
Esti
mateInMemory
StatsHistorySize
()
const
{
size_t
size_total
=
sizeof
(
std
::
map
<
uint64_t
,
std
::
map
<
std
::
string
,
uint64_t
>>
);
if
(
stats_history_
.
size
()
==
0
)
return
size_total
;
...
...
@@ -814,12 +823,40 @@ void DBImpl::PersistStats() {
stats_history_size_limit
=
mutable_db_options_
.
stats_history_buffer_size
;
}
// TODO(Zhongyi): also persist immutable_db_options_.statistics
{
std
::
map
<
std
::
string
,
uint64_t
>
stats_map
;
if
(
!
statistics
->
getTickerMap
(
&
stats_map
))
{
return
;
std
::
map
<
std
::
string
,
uint64_t
>
stats_map
;
if
(
!
statistics
->
getTickerMap
(
&
stats_map
))
{
return
;
}
if
(
immutable_db_options_
.
persist_stats_to_disk
)
{
WriteBatch
batch
;
if
(
stats_slice_initialized_
)
{
for
(
const
auto
&
stat
:
stats_map
)
{
char
key
[
100
];
int
length
=
EncodePersistentStatsKey
(
now_seconds
,
stat
.
first
,
100
,
key
);
// calculate the delta from last time
if
(
stats_slice_
.
find
(
stat
.
first
)
!=
stats_slice_
.
end
())
{
uint64_t
delta
=
stat
.
second
-
stats_slice_
[
stat
.
first
];
batch
.
Put
(
persist_stats_cf_handle_
,
Slice
(
key
,
std
::
min
(
100
,
length
)),
ToString
(
delta
));
}
}
}
stats_slice_initialized_
=
true
;
std
::
swap
(
stats_slice_
,
stats_map
);
WriteOptions
wo
;
wo
.
low_pri
=
true
;
wo
.
no_slowdown
=
true
;
wo
.
sync
=
false
;
Status
s
=
Write
(
wo
,
&
batch
);
if
(
!
s
.
ok
())
{
ROCKS_LOG_INFO
(
immutable_db_options_
.
info_log
,
"Writing to persistent stats CF failed -- %s
\n
"
,
s
.
ToString
().
c_str
());
}
// TODO(Zhongyi): add purging for persisted data
}
else
{
InstrumentedMutexLock
l
(
&
stats_history_mutex_
);
// calculate the delta from last time
if
(
stats_slice_initialized_
)
{
...
...
@@ -829,17 +866,19 @@ void DBImpl::PersistStats() {
stats_delta
[
stat
.
first
]
=
stat
.
second
-
stats_slice_
[
stat
.
first
];
}
}
stats_history_
[
now_
micro
s
]
=
stats_delta
;
stats_history_
[
now_
second
s
]
=
stats_delta
;
}
stats_slice_initialized_
=
true
;
std
::
swap
(
stats_slice_
,
stats_map
);
TEST_SYNC_POINT
(
"DBImpl::PersistStats:StatsCopied"
);
// delete older stats snapshots to control memory consumption
bool
purge_needed
=
EstiamteStatsHistorySize
()
>
stats_history_size_limit
;
bool
purge_needed
=
EstimateInMemoryStatsHistorySize
()
>
stats_history_size_limit
;
while
(
purge_needed
&&
!
stats_history_
.
empty
())
{
stats_history_
.
erase
(
stats_history_
.
begin
());
purge_needed
=
EstiamteStatsHistorySize
()
>
stats_history_size_limit
;
purge_needed
=
EstimateInMemoryStatsHistorySize
()
>
stats_history_size_limit
;
}
}
...
...
@@ -874,8 +913,13 @@ Status DBImpl::GetStatsHistory(
if
(
!
stats_iterator
)
{
return
Status
::
InvalidArgument
(
"stats_iterator not preallocated."
);
}
stats_iterator
->
reset
(
new
InMemoryStatsHistoryIterator
(
start_time
,
end_time
,
this
));
if
(
immutable_db_options_
.
persist_stats_to_disk
)
{
stats_iterator
->
reset
(
new
PersistentStatsHistoryIterator
(
start_time
,
end_time
,
this
));
}
else
{
stats_iterator
->
reset
(
new
InMemoryStatsHistoryIterator
(
start_time
,
end_time
,
this
));
}
return
(
*
stats_iterator
)
->
status
();
}
void
DBImpl
::
ScheduleGCTTL
()
{
...
...
@@ -1131,14 +1175,14 @@ Status DBImpl::SetDBOptions(
stats_dump_scheduler_
->
Unregister
(
this
);
mutex_
.
Lock
();
}
if
(
new_options
.
stats_dump_period_sec
>
0
||
new_options
.
stats_persist_period_sec
>
0
)
{
}
if
(
new_options
.
stats_persist_period_sec
!=
mutable_db_options_
.
stats_persist_period_sec
)
{
mutex_
.
Unlock
();
stats_dump_scheduler_
->
Register
(
this
,
new_options
.
stats_dump_period_sec
,
new_options
.
stats_persist_period_sec
);
mutex_
.
Lock
();
}
}
write_controller_
.
set_max_delayed_write_rate
(
new_options
.
delayed_write_rate
);
...
...
@@ -1561,6 +1605,10 @@ ColumnFamilyHandle* DBImpl::DefaultColumnFamily() const {
return
default_cf_handle_
;
}
ColumnFamilyHandle
*
DBImpl
::
PersistentStatsColumnFamily
()
const
{
return
persist_stats_cf_handle_
;
}
Status
DBImpl
::
Get
(
const
ReadOptions
&
read_options
,
ColumnFamilyHandle
*
column_family
,
const
Slice
&
key
,
LazyBuffer
*
value
)
{
...
...
db/db_impl.h
浏览文件 @
85df8fbb
...
...
@@ -68,12 +68,12 @@ class Arena;
class
ArenaWrappedDBIter
;
class
InMemoryStatsHistoryIterator
;
class
MemTable
;
class
PersistentStatsHistoryIterator
;
class
StatsDumpScheduler
;
#ifndef NDEBUG
class
StatsDumpTestScheduler
;
#endif // !NDEBUG
class
TableCache
;
class
Version
;
class
VersionEdit
;
...
...
@@ -255,6 +255,17 @@ class DBImpl : public DB {
virtual
bool
SetPreserveDeletesSequenceNumber
(
SequenceNumber
seqnum
)
override
;
virtual
Status
GetDbIdentity
(
std
::
string
&
identity
)
const
override
;
ColumnFamilyHandle
*
DefaultColumnFamily
()
const
override
;
ColumnFamilyHandle
*
PersistentStatsColumnFamily
()
const
;
virtual
Status
Close
()
override
;
Status
GetStatsHistory
(
uint64_t
start_time
,
uint64_t
end_time
,
std
::
unique_ptr
<
StatsHistoryIterator
>*
stats_iterator
)
override
;
#ifndef ROCKSDB_LITE
using
DB
::
ResetStats
;
virtual
Status
ResetStats
()
override
;
...
...
@@ -782,9 +793,19 @@ class DBImpl : public DB {
VersionSet
*
TEST_GetVersionSet
()
const
{
return
versions_
.
get
();
}
<<<<<<<
HEAD
:
db
/
db_impl
.
h
#ifndef ROCKSDB_LITE
StatsDumpTestScheduler
*
TEST_GetStatsDumpScheduler
()
const
;
#endif // !ROCKSDB_LITE
=======
int
TEST_BGCompactionsAllowed
()
const
;
int
TEST_BGFlushesAllowed
()
const
;
size_t
TEST_GetWalPreallocateBlockSize
(
uint64_t
write_buffer_size
)
const
;
void
TEST_WaitForDumpStatsRun
(
std
::
function
<
void
()
>
callback
)
const
;
void
TEST_WaitForPersistStatsRun
(
std
::
function
<
void
()
>
callback
)
const
;
bool
TEST_IsPersistentStatsEnabled
()
const
;
size_t
TEST_EstimateInMemoryStatsHistorySize
()
const
;
>>>>>>>
671
d15cbd
...
Persistent
Stats
:
persist
stats
history
to
disk
(
#
5046
)
:
db
/
db_impl
/
db_impl
.
h
#endif // NDEBUG
protected:
...
...
@@ -910,6 +931,9 @@ class DBImpl : public DB {
friend
class
CompactedDBImpl
;
friend
class
DBTest_ConcurrentFlushWAL_Test
;
friend
class
DBTest_MixedSlowdownOptionsStop_Test
;
friend
class
DBCompactionTest_CompactBottomLevelFilesWithDeletions_Test
;
friend
class
DBCompactionTest_CompactionDuringShutdown_Test
;
friend
class
StatsHistoryTest_PersistentStatsCreateColumnFamilies_Test
;
#ifndef NDEBUG
friend
class
DBTest2_ReadCallbackTest_Test
;
friend
class
WriteCallbackTest_WriteWithCallbackTest_Test
;
...
...
@@ -943,6 +967,21 @@ class DBImpl : public DB {
bool
read_only
=
false
,
bool
error_if_log_file_exist
=
false
,
bool
error_if_data_exists_in_logs
=
false
);
// Initialize the built-in column family for persistent stats. Depending on
// whether on-disk persistent stats have been enabled before, it may either
// create a new column family and column family handle or just a column family
// handle.
// Required: DB mutex held
Status
InitPersistStatsColumnFamily
();
// Persistent Stats column family has two format version key which are used
// for compatibility check. Write format version if it's created for the
// first time, read format version and check compatibility if recovering
// from disk. This function requires DB mutex held at entrance but may
// release and re-acquire DB mutex in the process.
// Required: DB mutex held
Status
PersistentStatsProcessFormatVersion
();
Status
ResumeImpl
();
void
MaybeIgnoreError
(
Status
*
s
)
const
;
...
...
@@ -1198,7 +1237,7 @@ class DBImpl : public DB {
void
PrintStatistics
();
size_t
Esti
amte
StatsHistorySize
()
const
;
size_t
Esti
mateInMemory
StatsHistorySize
()
const
;
// persist stats to column family "_persistent_stats"
void
PersistStats
();
...
...
@@ -1337,6 +1376,9 @@ class DBImpl : public DB {
// true for some prefix of logs_
bool
getting_synced
=
false
;
};
ColumnFamilyHandleImpl
*
persist_stats_cf_handle_
;
bool
persistent_stats_cfd_exists_
=
true
;
// Without two_write_queues, read and writes to alive_log_files_ are
// protected by mutex_. However since back() is never popped, and push_back()
// is done only from write_thread_, the same thread can access the item
...
...
db/db_impl_debug.cc
浏览文件 @
85df8fbb
...
...
@@ -265,8 +265,8 @@ StatsDumpTestScheduler* DBImpl::TEST_GetStatsDumpScheduler() const {
}
#endif // !ROCKSDB_LITE
size_t
DBImpl
::
TEST_Esti
amte
StatsHistorySize
()
const
{
return
Esti
amte
StatsHistorySize
();
size_t
DBImpl
::
TEST_Esti
mateInMemory
StatsHistorySize
()
const
{
return
Esti
mateInMemory
StatsHistorySize
();
}
}
// namespace TERARKDB_NAMESPACE
#endif // NDEBUG
db/db_impl_open.cc
浏览文件 @
85df8fbb
...
...
@@ -16,6 +16,7 @@
#include "db/builder.h"
#include "db/error_handler.h"
#include "db/map_builder.h"
#include "monitoring/persistent_stats_history.h"
#include "options/options_helper.h"
#include "rocksdb/wal_filter.h"
#include "table/block_based_table_factory.h"
...
...
@@ -23,6 +24,7 @@
#include "util/rate_limiter.h"
#include "util/sst_file_manager_impl.h"
#include "util/sync_point.h"
#include "util/string_util.h"
#if !defined(_MSC_VER) && !defined(__APPLE__)
#include <sys/unistd.h>
#include <table/terark_zip_table.h>
...
...
@@ -405,6 +407,7 @@ Status DBImpl::Recover(
}
Status
s
=
versions_
->
Recover
(
column_families
,
read_only
);
if
(
immutable_db_options_
.
paranoid_checks
&&
s
.
ok
())
{
s
=
CheckConsistency
(
read_only
);
}
...
...
@@ -416,6 +419,10 @@ Status DBImpl::Recover(
}
}
}
// DB mutex is already held
if
(
s
.
ok
()
&&
immutable_db_options_
.
persist_stats_to_disk
)
{
s
=
InitPersistStatsColumnFamily
();
}
// Initial max_total_in_memory_state_ before recovery logs. Log recovery
// may check this value to decide whether to flush.
...
...
@@ -431,6 +438,8 @@ Status DBImpl::Recover(
default_cf_handle_
=
new
ColumnFamilyHandleImpl
(
versions_
->
GetColumnFamilySet
()
->
GetDefault
(),
this
,
&
mutex_
);
default_cf_internal_stats_
=
default_cf_handle_
->
cfd
()
->
internal_stats
();
// TODO(Zhongyi): handle single_column_family_mode_ when
// persistent_stats is enabled
single_column_family_mode_
=
versions_
->
GetColumnFamilySet
()
->
NumberOfColumnFamilies
()
==
1
;
...
...
@@ -538,6 +547,98 @@ Status DBImpl::Recover(
return
s
;
}
Status
DBImpl
::
PersistentStatsProcessFormatVersion
()
{
mutex_
.
AssertHeld
();
Status
s
;
// persist version when stats CF doesn't exist
bool
should_persist_format_version
=
!
persistent_stats_cfd_exists_
;
mutex_
.
Unlock
();
if
(
persistent_stats_cfd_exists_
)
{
// Check persistent stats format version compatibility. Drop and recreate
// persistent stats CF if format version is incompatible
uint64_t
format_version_recovered
=
0
;
Status
s_format
=
DecodePersistentStatsVersionNumber
(
this
,
StatsVersionKeyType
::
kFormatVersion
,
&
format_version_recovered
);
uint64_t
compatible_version_recovered
=
0
;
Status
s_compatible
=
DecodePersistentStatsVersionNumber
(
this
,
StatsVersionKeyType
::
kCompatibleVersion
,
&
compatible_version_recovered
);
// abort reading from existing stats CF if any of following is true:
// 1. failed to read format version or compatible version from disk
// 2. sst's format version is greater than current format version, meaning
// this sst is encoded with a newer RocksDB release, and current compatible
// version is below the sst's compatible version
if
(
!
s_format
.
ok
()
||
!
s_compatible
.
ok
()
||
(
kStatsCFCurrentFormatVersion
<
format_version_recovered
&&
kStatsCFCompatibleFormatVersion
<
compatible_version_recovered
))
{
if
(
!
s_format
.
ok
()
||
!
s_compatible
.
ok
())
{
ROCKS_LOG_INFO
(
immutable_db_options_
.
info_log
,
"Reading persistent stats version key failed. Format key: %s, "
"compatible key: %s"
,
s_format
.
ToString
().
c_str
(),
s_compatible
.
ToString
().
c_str
());
}
else
{
ROCKS_LOG_INFO
(
immutable_db_options_
.
info_log
,
"Disable persistent stats due to corrupted or incompatible format "
"version
\n
"
);
}
DropColumnFamily
(
persist_stats_cf_handle_
);
DestroyColumnFamilyHandle
(
persist_stats_cf_handle_
);
ColumnFamilyHandle
*
handle
=
nullptr
;
ColumnFamilyOptions
cfo
;
OptimizeForPersistentStats
(
&
cfo
);
s
=
CreateColumnFamily
(
cfo
,
kPersistentStatsColumnFamilyName
,
&
handle
);
persist_stats_cf_handle_
=
static_cast
<
ColumnFamilyHandleImpl
*>
(
handle
);
// should also persist version here because old stats CF is discarded
should_persist_format_version
=
true
;
}
}
if
(
s
.
ok
()
&&
should_persist_format_version
)
{
// Persistent stats CF being created for the first time, need to write
// format version key
WriteBatch
batch
;
batch
.
Put
(
persist_stats_cf_handle_
,
kFormatVersionKeyString
,
ToString
(
kStatsCFCurrentFormatVersion
));
batch
.
Put
(
persist_stats_cf_handle_
,
kCompatibleVersionKeyString
,
ToString
(
kStatsCFCompatibleFormatVersion
));
WriteOptions
wo
;
wo
.
low_pri
=
true
;
wo
.
no_slowdown
=
true
;
wo
.
sync
=
false
;
s
=
Write
(
wo
,
&
batch
);
}
mutex_
.
Lock
();
return
s
;
}
Status
DBImpl
::
InitPersistStatsColumnFamily
()
{
mutex_
.
AssertHeld
();
assert
(
!
persist_stats_cf_handle_
);
ColumnFamilyData
*
persistent_stats_cfd
=
versions_
->
GetColumnFamilySet
()
->
GetColumnFamily
(
kPersistentStatsColumnFamilyName
);
persistent_stats_cfd_exists_
=
persistent_stats_cfd
!=
nullptr
;
Status
s
;
if
(
persistent_stats_cfd
!=
nullptr
)
{
// We are recovering from a DB which already contains persistent stats CF,
// the CF is already created in VersionSet::ApplyOneVersionEdit, but
// column family handle was not. Need to explicitly create handle here.
persist_stats_cf_handle_
=
new
ColumnFamilyHandleImpl
(
persistent_stats_cfd
,
this
,
&
mutex_
);
}
else
{
mutex_
.
Unlock
();
ColumnFamilyHandle
*
handle
=
nullptr
;
ColumnFamilyOptions
cfo
;
OptimizeForPersistentStats
(
&
cfo
);
s
=
CreateColumnFamily
(
cfo
,
kPersistentStatsColumnFamilyName
,
&
handle
);
persist_stats_cf_handle_
=
static_cast
<
ColumnFamilyHandleImpl
*>
(
handle
);
mutex_
.
Lock
();
}
return
s
;
}
// REQUIRES: log_numbers are sorted in ascending order
Status
DBImpl
::
RecoverLogFiles
(
const
std
::
vector
<
uint64_t
>&
log_numbers
,
SequenceNumber
*
next_sequence
,
bool
read_only
)
{
...
...
@@ -1119,12 +1220,23 @@ Status DB::Open(const Options& options, const std::string& dbname, DB** dbptr) {
std
::
vector
<
ColumnFamilyDescriptor
>
column_families
;
column_families
.
push_back
(
ColumnFamilyDescriptor
(
kDefaultColumnFamilyName
,
cf_options
));
if
(
db_options
.
persist_stats_to_disk
)
{
column_families
.
push_back
(
ColumnFamilyDescriptor
(
kPersistentStatsColumnFamilyName
,
cf_options
));
}
std
::
vector
<
ColumnFamilyHandle
*>
handles
;
Status
s
=
DB
::
Open
(
db_options
,
dbname
,
column_families
,
&
handles
,
dbptr
);
if
(
s
.
ok
())
{
assert
(
handles
.
size
()
==
1
);
if
(
db_options
.
persist_stats_to_disk
)
{
assert
(
handles
.
size
()
==
2
);
}
else
{
assert
(
handles
.
size
()
==
1
);
}
// i can delete the handle since DBImpl is always holding a reference to
// default column family
if
(
db_options
.
persist_stats_to_disk
&&
handles
[
1
]
!=
nullptr
)
{
delete
handles
[
1
];
}
delete
handles
[
0
];
}
return
s
;
...
...
@@ -1298,6 +1410,10 @@ Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname,
s
=
impl
->
directories_
.
GetDbDir
()
->
Fsync
();
}
}
if
(
s
.
ok
()
&&
impl
->
immutable_db_options_
.
persist_stats_to_disk
)
{
// try to read format version but no need to fail Open() even if it fails
s
=
impl
->
PersistentStatsProcessFormatVersion
();
}
if
(
s
.
ok
())
{
for
(
auto
cfd
:
*
impl
->
versions_
->
GetColumnFamilySet
())
{
...
...
db/db_options_test.cc
浏览文件 @
85df8fbb
...
...
@@ -537,6 +537,7 @@ TEST_F(DBOptionsTest, SetStatsDumpPeriodSec) {
Close
();
}
<<<<<<<
HEAD
TEST_F
(
DBOptionsTest
,
RunStatsDumpPeriodSec
)
{
Options
options
;
options
.
create_if_missing
=
true
;
...
...
@@ -681,6 +682,7 @@ TEST_F(DBOptionsTest, SetOptionsStatsPersistPeriodSec) {
ASSERT_EQ
(
12345
,
dbfull
()
->
GetDBOptions
().
stats_persist_period_sec
);
}
<<<<<<<
HEAD
TEST_F
(
DBOptionsTest
,
GetStatsHistory
)
{
Options
options
;
options
.
create_if_missing
=
true
;
...
...
db/version_set.cc
浏览文件 @
85df8fbb
...
...
@@ -17,6 +17,8 @@
#include <stdio.h>
#include <algorithm>
#include <array>
#include <cinttypes>
#include <list>
#include <map>
#include <set>
...
...
@@ -35,6 +37,7 @@
#include "db/version_builder.h"
#include "monitoring/file_read_sample.h"
#include "monitoring/perf_context_imp.h"
#include "monitoring/persistent_stats_history.h"
#include "rocksdb/env.h"
#include "rocksdb/merge_operator.h"
#include "rocksdb/write_buffer_manager.h"
...
...
@@ -3510,11 +3513,23 @@ Status VersionSet::ApplyOneVersionEdit(
edit
.
column_family_name_
);
}
auto
cf_options
=
name_to_options
.
find
(
edit
.
column_family_name_
);
if
(
cf_options
==
name_to_options
.
end
())
{
// implicitly add persistent_stats column family without requiring user
// to specify
bool
is_persistent_stats_column_family
=
edit
.
column_family_name_
.
compare
(
kPersistentStatsColumnFamilyName
)
==
0
;
if
(
cf_options
==
name_to_options
.
end
()
&&
!
is_persistent_stats_column_family
)
{
column_families_not_found
.
insert
(
{
edit
.
column_family_
,
edit
.
column_family_name_
});
}
else
{
cfd
=
CreateColumnFamily
(
cf_options
->
second
,
&
edit
);
// recover persistent_stats CF from a DB that already contains it
if
(
is_persistent_stats_column_family
)
{
ColumnFamilyOptions
cfo
;
OptimizeForPersistentStats
(
&
cfo
);
cfd
=
CreateColumnFamily
(
cfo
,
&
edit
);
}
else
{
cfd
=
CreateColumnFamily
(
cf_options
->
second
,
&
edit
);
}
cfd
->
set_initialized
();
builders
.
insert
(
{
edit
.
column_family_
,
new
BaseReferencedVersionBuilder
(
cfd
)});
...
...
include/rocksdb/db.h
浏览文件 @
85df8fbb
...
...
@@ -72,6 +72,7 @@ class TraceWriter;
using
std
::
unique_ptr
;
extern
const
std
::
string
kDefaultColumnFamilyName
;
extern
const
std
::
string
kPersistentStatsColumnFamilyName
;
struct
ColumnFamilyDescriptor
{
std
::
string
name
;
ColumnFamilyOptions
options
;
...
...
@@ -1275,10 +1276,11 @@ class DB {
// Needed for StackableDB
virtual
DB
*
GetRootDB
()
{
return
this
;
}
// Given a time window, return an iterator for accessing stats history
// User is responsible for deleting StatsHistoryIterator after use
virtual
Status
GetStatsHistory
(
uint64_t
/*start_time*/
,
uint64_t
/*end_time*/
,
// Given a window [start_time, end_time), setup a StatsHistoryIterator
// to access stats history. Note the start_time and end_time are epoch
// time measured in seconds, and end_time is an exclusive bound.
virtual
Status
GetStatsHistory
(
uint64_t
/*start_time*/
,
uint64_t
/*end_time*/
,
std
::
unique_ptr
<
StatsHistoryIterator
>*
/*stats_iterator*/
)
{
return
Status
::
NotSupported
(
"GetStatsHistory() is not implemented."
);
}
...
...
include/rocksdb/options.h
浏览文件 @
85df8fbb
...
...
@@ -737,6 +737,18 @@ struct DBOptions {
// Default: 600
unsigned
int
stats_persist_period_sec
=
600
;
// If true, automatically persist stats to a hidden column family (column
// family name: ___rocksdb_stats_history___) every
// stats_persist_period_sec seconds; otherwise, write to an in-memory
// struct. User can query through `GetStatsHistory` API.
// If user attempts to create a column family with the same name on a DB
// which have previously set persist_stats_to_disk to true, the column family
// creation will fail, but the hidden column family will survive, as well as
// the previously persisted statistics.
// When peristing stats to disk, the stat name will be limited at 100 bytes.
// Default: false
bool
persist_stats_to_disk
=
false
;
// if not zero, periodically take stats snapshots and store in memory, the
// memory size for stats snapshots is capped at stats_history_buffer_size
// Default: 1MB
...
...
include/rocksdb/stats_history.h
浏览文件 @
85df8fbb
...
...
@@ -31,10 +31,12 @@ class StatsHistoryIterator {
// REQUIRES: Valid()
virtual
void
Next
()
=
0
;
// Return the time stamp (in
micro
seconds) when stats history is recorded.
// Return the time stamp (in seconds) when stats history is recorded.
// REQUIRES: Valid()
virtual
uint64_t
GetStatsTime
()
const
=
0
;
virtual
int
GetFormatVersion
()
const
{
return
-
1
;
}
// Return the current stats history as an std::map which specifies the
// mapping from stats name to stats value . The underlying storage
// for the returned map is valid only until the next modification of
...
...
db
/in_memory_stats_history.cc
→
monitoring
/in_memory_stats_history.cc
浏览文件 @
85df8fbb
...
...
@@ -7,7 +7,7 @@
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "db/db_impl.h"
#include "
db
/in_memory_stats_history.h"
#include "
monitoring
/in_memory_stats_history.h"
namespace
rocksdb
{
...
...
db
/in_memory_stats_history.h
→
monitoring
/in_memory_stats_history.h
浏览文件 @
85df8fbb
...
...
@@ -14,6 +14,8 @@ namespace rocksdb {
class
InMemoryStatsHistoryIterator
final
:
public
StatsHistoryIterator
{
public:
// Setup InMemoryStatsHistoryIterator to return stats snapshots between
// seconds timestamps [start_time, end_time)
InMemoryStatsHistoryIterator
(
uint64_t
start_time
,
uint64_t
end_time
,
DBImpl
*
db_impl
)
:
start_time_
(
start_time
),
...
...
monitoring/persistent_stats_history.cc
0 → 100644
浏览文件 @
85df8fbb
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "monitoring/persistent_stats_history.h"
#include <cstring>
#include <string>
#include <utility>
#include "db/db_impl/db_impl.h"
#include "port/likely.h"
#include "util/string_util.h"
namespace
rocksdb
{
// 10 digit seconds timestamp => [Sep 9, 2001 ~ Nov 20, 2286]
const
int
kNowSecondsStringLength
=
10
;
const
std
::
string
kFormatVersionKeyString
=
"__persistent_stats_format_version__"
;
const
std
::
string
kCompatibleVersionKeyString
=
"__persistent_stats_compatible_version__"
;
// Every release maintains two versions numbers for persistents stats: Current
// format version and compatible format version. Current format version
// designates what type of encoding will be used when writing to stats CF;
// compatible format version designates the minimum format version that
// can decode the stats CF encoded using the current format version.
const
uint64_t
kStatsCFCurrentFormatVersion
=
1
;
const
uint64_t
kStatsCFCompatibleFormatVersion
=
1
;
Status
DecodePersistentStatsVersionNumber
(
DBImpl
*
db
,
StatsVersionKeyType
type
,
uint64_t
*
version_number
)
{
if
(
type
>=
StatsVersionKeyType
::
kKeyTypeMax
)
{
return
Status
::
InvalidArgument
(
"Invalid stats version key type provided"
);
}
std
::
string
key
;
if
(
type
==
StatsVersionKeyType
::
kFormatVersion
)
{
key
=
kFormatVersionKeyString
;
}
else
if
(
type
==
StatsVersionKeyType
::
kCompatibleVersion
)
{
key
=
kCompatibleVersionKeyString
;
}
ReadOptions
options
;
options
.
verify_checksums
=
true
;
std
::
string
result
;
Status
s
=
db
->
Get
(
options
,
db
->
PersistentStatsColumnFamily
(),
key
,
&
result
);
if
(
!
s
.
ok
()
||
result
.
empty
())
{
return
Status
::
NotFound
(
"Persistent stats version key "
+
key
+
" not found."
);
}
// read version_number but do nothing in current version
*
version_number
=
ParseUint64
(
result
);
return
Status
::
OK
();
}
int
EncodePersistentStatsKey
(
uint64_t
now_seconds
,
const
std
::
string
&
key
,
int
size
,
char
*
buf
)
{
char
timestamp
[
kNowSecondsStringLength
+
1
];
// make time stamp string equal in length to allow sorting by time
snprintf
(
timestamp
,
sizeof
(
timestamp
),
"%010d"
,
static_cast
<
int
>
(
now_seconds
));
timestamp
[
kNowSecondsStringLength
]
=
'\0'
;
return
snprintf
(
buf
,
size
,
"%s#%s"
,
timestamp
,
key
.
c_str
());
}
void
OptimizeForPersistentStats
(
ColumnFamilyOptions
*
cfo
)
{
cfo
->
write_buffer_size
=
2
<<
20
;
cfo
->
target_file_size_base
=
2
*
1048576
;
cfo
->
max_bytes_for_level_base
=
10
*
1048576
;
cfo
->
snap_refresh_nanos
=
0
;
cfo
->
soft_pending_compaction_bytes_limit
=
256
*
1048576
;
cfo
->
hard_pending_compaction_bytes_limit
=
1073741824ul
;
cfo
->
compression
=
kNoCompression
;
}
PersistentStatsHistoryIterator
::~
PersistentStatsHistoryIterator
()
{}
bool
PersistentStatsHistoryIterator
::
Valid
()
const
{
return
valid_
;
}
Status
PersistentStatsHistoryIterator
::
status
()
const
{
return
status_
;
}
void
PersistentStatsHistoryIterator
::
Next
()
{
// increment start_time by 1 to avoid infinite loop
AdvanceIteratorByTime
(
GetStatsTime
()
+
1
,
end_time_
);
}
uint64_t
PersistentStatsHistoryIterator
::
GetStatsTime
()
const
{
return
time_
;
}
const
std
::
map
<
std
::
string
,
uint64_t
>&
PersistentStatsHistoryIterator
::
GetStatsMap
()
const
{
return
stats_map_
;
}
std
::
pair
<
uint64_t
,
std
::
string
>
parseKey
(
const
Slice
&
key
,
uint64_t
start_time
)
{
std
::
pair
<
uint64_t
,
std
::
string
>
result
;
std
::
string
key_str
=
key
.
ToString
();
std
::
string
::
size_type
pos
=
key_str
.
find
(
"#"
);
// TODO(Zhongyi): add counters to track parse failures?
if
(
pos
==
std
::
string
::
npos
)
{
result
.
first
=
port
::
kMaxUint64
;
result
.
second
.
clear
();
}
else
{
uint64_t
parsed_time
=
ParseUint64
(
key_str
.
substr
(
0
,
pos
));
// skip entries with timestamp smaller than start_time
if
(
parsed_time
<
start_time
)
{
result
.
first
=
port
::
kMaxUint64
;
result
.
second
=
""
;
}
else
{
result
.
first
=
parsed_time
;
std
::
string
key_resize
=
key_str
.
substr
(
pos
+
1
);
result
.
second
=
key_resize
;
}
}
return
result
;
}
// advance the iterator to the next time between [start_time, end_time)
// if success, update time_ and stats_map_ with new_time and stats_map
void
PersistentStatsHistoryIterator
::
AdvanceIteratorByTime
(
uint64_t
start_time
,
uint64_t
end_time
)
{
// try to find next entry in stats_history_ map
if
(
db_impl_
!=
nullptr
)
{
ReadOptions
ro
;
Iterator
*
iter
=
db_impl_
->
NewIterator
(
ro
,
db_impl_
->
PersistentStatsColumnFamily
());
char
timestamp
[
kNowSecondsStringLength
+
1
];
snprintf
(
timestamp
,
sizeof
(
timestamp
),
"%010d"
,
static_cast
<
int
>
(
std
::
max
(
time_
,
start_time
)));
timestamp
[
kNowSecondsStringLength
]
=
'\0'
;
iter
->
Seek
(
timestamp
);
// no more entries with timestamp >= start_time is found or version key
// is found to be incompatible
if
(
!
iter
->
Valid
())
{
valid_
=
false
;
delete
iter
;
return
;
}
time_
=
parseKey
(
iter
->
key
(),
start_time
).
first
;
valid_
=
true
;
// check parsed time and invalid if it exceeds end_time
if
(
time_
>
end_time
)
{
valid_
=
false
;
delete
iter
;
return
;
}
// find all entries with timestamp equal to time_
std
::
map
<
std
::
string
,
uint64_t
>
new_stats_map
;
std
::
pair
<
uint64_t
,
std
::
string
>
kv
;
for
(;
iter
->
Valid
();
iter
->
Next
())
{
kv
=
parseKey
(
iter
->
key
(),
start_time
);
if
(
kv
.
first
!=
time_
)
{
break
;
}
if
(
kv
.
second
.
compare
(
kFormatVersionKeyString
)
==
0
)
{
continue
;
}
new_stats_map
[
kv
.
second
]
=
ParseUint64
(
iter
->
value
().
ToString
());
}
stats_map_
.
swap
(
new_stats_map
);
delete
iter
;
}
else
{
valid_
=
false
;
}
}
}
// namespace rocksdb
monitoring/persistent_stats_history.h
0 → 100644
浏览文件 @
85df8fbb
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#pragma once
#include "db/db_impl/db_impl.h"
#include "rocksdb/stats_history.h"
namespace
rocksdb
{
extern
const
std
::
string
kFormatVersionKeyString
;
extern
const
std
::
string
kCompatibleVersionKeyString
;
extern
const
uint64_t
kStatsCFCurrentFormatVersion
;
extern
const
uint64_t
kStatsCFCompatibleFormatVersion
;
enum
StatsVersionKeyType
:
uint32_t
{
kFormatVersion
=
1
,
kCompatibleVersion
=
2
,
kKeyTypeMax
=
3
};
// Read the version number from persitent stats cf depending on type provided
// stores the version number in `*version_number`
// returns Status::OK() on success, or other status code on failure
Status
DecodePersistentStatsVersionNumber
(
DBImpl
*
db
,
StatsVersionKeyType
type
,
uint64_t
*
version_number
);
// Encode timestamp and stats key into buf
// Format: timestamp(10 digit) + '#' + key
// Total length of encoded key will be capped at 100 bytes
int
EncodePersistentStatsKey
(
uint64_t
timestamp
,
const
std
::
string
&
key
,
int
size
,
char
*
buf
);
void
OptimizeForPersistentStats
(
ColumnFamilyOptions
*
cfo
);
class
PersistentStatsHistoryIterator
final
:
public
StatsHistoryIterator
{
public:
PersistentStatsHistoryIterator
(
uint64_t
start_time
,
uint64_t
end_time
,
DBImpl
*
db_impl
)
:
time_
(
0
),
start_time_
(
start_time
),
end_time_
(
end_time
),
valid_
(
true
),
db_impl_
(
db_impl
)
{
AdvanceIteratorByTime
(
start_time_
,
end_time_
);
}
~
PersistentStatsHistoryIterator
()
override
;
bool
Valid
()
const
override
;
Status
status
()
const
override
;
void
Next
()
override
;
uint64_t
GetStatsTime
()
const
override
;
const
std
::
map
<
std
::
string
,
uint64_t
>&
GetStatsMap
()
const
override
;
private:
// advance the iterator to the next stats history record with timestamp
// between [start_time, end_time)
void
AdvanceIteratorByTime
(
uint64_t
start_time
,
uint64_t
end_time
);
// No copying allowed
PersistentStatsHistoryIterator
(
const
PersistentStatsHistoryIterator
&
)
=
delete
;
void
operator
=
(
const
PersistentStatsHistoryIterator
&
)
=
delete
;
PersistentStatsHistoryIterator
(
PersistentStatsHistoryIterator
&&
)
=
delete
;
PersistentStatsHistoryIterator
&
operator
=
(
PersistentStatsHistoryIterator
&&
)
=
delete
;
uint64_t
time_
;
uint64_t
start_time_
;
uint64_t
end_time_
;
std
::
map
<
std
::
string
,
uint64_t
>
stats_map_
;
Status
status_
;
bool
valid_
;
DBImpl
*
db_impl_
;
};
}
// namespace rocksdb
monitoring/stats_history_test.cc
浏览文件 @
85df8fbb
此差异已折叠。
点击以展开。
options/db_options.cc
浏览文件 @
85df8fbb
...
...
@@ -93,7 +93,8 @@ ImmutableDBOptions::ImmutableDBOptions(const DBOptions& options)
two_write_queues
(
options
.
two_write_queues
),
manual_wal_flush
(
options
.
manual_wal_flush
),
atomic_flush
(
options
.
atomic_flush
),
avoid_unnecessary_blocking_io
(
options
.
avoid_unnecessary_blocking_io
)
{
avoid_unnecessary_blocking_io
(
options
.
avoid_unnecessary_blocking_io
),
persist_stats_to_disk
(
options
.
persist_stats_to_disk
)
{
}
void
ImmutableDBOptions
::
Dump
(
Logger
*
log
)
const
{
...
...
@@ -247,6 +248,8 @@ void ImmutableDBOptions::Dump(Logger* log) const {
atomic_flush
);
ROCKS_LOG_HEADER
(
log
,
" Options.avoid_unnecessary_blocking_io: %d"
,
avoid_unnecessary_blocking_io
);
ROCKS_LOG_HEADER
(
log
,
" Options.persist_stats_to_disk: %u"
,
persist_stats_to_disk
);
}
MutableDBOptions
::
MutableDBOptions
()
...
...
options/db_options.h
浏览文件 @
85df8fbb
...
...
@@ -87,6 +87,7 @@ struct ImmutableDBOptions {
bool
manual_wal_flush
;
bool
atomic_flush
;
bool
avoid_unnecessary_blocking_io
;
bool
persist_stats_to_disk
;
};
struct
MutableDBOptions
{
...
...
options/options_helper.cc
浏览文件 @
85df8fbb
...
...
@@ -90,6 +90,7 @@ DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options,
options
.
stats_dump_period_sec
=
mutable_db_options
.
stats_dump_period_sec
;
options
.
stats_persist_period_sec
=
mutable_db_options
.
stats_persist_period_sec
;
options
.
persist_stats_to_disk
=
immutable_db_options
.
persist_stats_to_disk
;
options
.
stats_history_buffer_size
=
mutable_db_options
.
stats_history_buffer_size
;
options
.
advise_random_on_open
=
immutable_db_options
.
advise_random_on_open
;
...
...
@@ -1626,6 +1627,10 @@ std::unordered_map<std::string, OptionTypeInfo>
{
offsetof
(
struct
DBOptions
,
stats_persist_period_sec
),
OptionType
::
kUInt
,
OptionVerificationType
::
kNormal
,
true
,
offsetof
(
struct
MutableDBOptions
,
stats_persist_period_sec
)}},
{
"persist_stats_to_disk"
,
{
offsetof
(
struct
DBOptions
,
persist_stats_to_disk
),
OptionType
::
kBoolean
,
OptionVerificationType
::
kNormal
,
false
,
offsetof
(
struct
ImmutableDBOptions
,
persist_stats_to_disk
)}},
{
"stats_history_buffer_size"
,
{
offsetof
(
struct
DBOptions
,
stats_history_buffer_size
),
OptionType
::
kSizeT
,
OptionVerificationType
::
kNormal
,
true
,
...
...
options/options_settable_test.cc
浏览文件 @
85df8fbb
...
...
@@ -275,6 +275,7 @@ TEST_F(OptionsSettableTest, DBOptionsAllFieldsSettable) {
"allow_mmap_writes=false;"
"stats_dump_period_sec=70127;"
"stats_persist_period_sec=54321;"
"persist_stats_to_disk=true;"
"stats_history_buffer_size=14159;"
"allow_fallocate=true;"
"allow_mmap_reads=false;"
...
...
options/options_test.cc
浏览文件 @
85df8fbb
...
...
@@ -135,6 +135,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
{
"skip_log_error_on_recovery"
,
"false"
},
{
"stats_dump_period_sec"
,
"46"
},
{
"stats_persist_period_sec"
,
"57"
},
{
"persist_stats_to_disk"
,
"false"
},
{
"stats_history_buffer_size"
,
"69"
},
{
"advise_random_on_open"
,
"true"
},
{
"allow_mmap_populate"
,
"true"
},
...
...
@@ -275,6 +276,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
ASSERT_EQ
(
new_db_opt
.
skip_log_error_on_recovery
,
false
);
ASSERT_EQ
(
new_db_opt
.
stats_dump_period_sec
,
46U
);
ASSERT_EQ
(
new_db_opt
.
stats_persist_period_sec
,
57U
);
ASSERT_EQ
(
new_db_opt
.
persist_stats_to_disk
,
false
);
ASSERT_EQ
(
new_db_opt
.
stats_history_buffer_size
,
69U
);
ASSERT_EQ
(
new_db_opt
.
advise_random_on_open
,
true
);
ASSERT_EQ
(
new_db_opt
.
allow_mmap_populate
,
true
);
...
...
src.mk
浏览文件 @
85df8fbb
...
...
@@ -36,7 +36,6 @@ LIB_SOURCES = \
db/flush_job.cc
\
db/flush_scheduler.cc
\
db/forward_iterator.cc
\
db/in_memory_stats_history.cc
\
db/internal_stats.cc
\
db/logs_with_prep_tracker.cc
\
db/log_reader.cc
\
...
...
@@ -82,10 +81,12 @@ LIB_SOURCES = \
memtable/write_buffer_manager.cc
\
monitoring/histogram.cc
\
monitoring/histogram_windowing.cc
\
monitoring/in_memory_stats_history.cc
\
monitoring/instrumented_mutex.cc
\
monitoring/iostats_context.cc
\
monitoring/perf_context.cc
\
monitoring/perf_level.cc
\
monitoring/persistent_stats_history.cc
\
monitoring/statistics.cc
\
monitoring/stats_dump_scheduler.cc
\
monitoring/thread_status_impl.cc
\
...
...
@@ -101,6 +102,7 @@ LIB_SOURCES = \
options/options_sanity_check.cc
\
port/port_posix.cc
\
port/stack_trace.cc
\
<<<<<<
< HEAD
table/adaptive_table_factory.cc
\
table/block.cc
\
table/block_based_filter_block.cc
\
...
...
@@ -117,6 +119,27 @@ LIB_SOURCES = \
table/data_block_hash_index.cc
\
table/data_block_footer.cc
\
table/flush_block_policy.cc
\
=======
table/adaptive/adaptive_table_factory.cc
\
table/block_based/block.cc
\
table/block_based/block_based_filter_block.cc
\
table/block_based/block_based_table_builder.cc
\
table/block_based/block_based_table_factory.cc
\
table/block_based/block_based_table_reader.cc
\
table/block_based/block_builder.cc
\
table/block_based/block_prefix_index.cc
\
table/block_based/data_block_hash_index.cc
\
table/block_based/data_block_footer.cc
\
table/block_based/flush_block_policy.cc
\
table/block_based/full_filter_block.cc
\
table/block_based/index_builder.cc
\
table/block_based/partitioned_filter_block.cc
\
table/block_fetcher.cc
\
table/bloom_block.cc
\
table/cuckoo/cuckoo_table_builder.cc
\
table/cuckoo/cuckoo_table_factory.cc
\
table/cuckoo/cuckoo_table_reader.cc
\
>>>>>>> 671d15cbd... Persistent Stats
:
persist stats history to disk (
#
5046)
table/format.cc
\
table/full_filter_block.cc
\
table/get_context.cc
\
...
...
@@ -249,21 +272,32 @@ LIB_SOURCES_ASM =
LIB_SOURCES_C
=
endif
TOOL_LIB_SOURCES
=
\
TOOL_LIB_SOURCES
=
\
tools/ldb_cmd.cc
\
tools/ldb_tool.cc
\
tools/sst_dump_tool.cc
\
<<<<<<<
HEAD
ANALYZER_LIB_SOURCES
=
\
tools/trace_analyzer_tool.cc
\
MOCK_LIB_SOURCES
=
\
table/mock_table.cc
\
util/fault_injection_test_env.cc
=======
ANALYZER_LIB_SOURCES
=
\
tools/block_cache_trace_analyzer.cc
\
tools/trace_analyzer_tool.cc
\
MOCK_LIB_SOURCES
=
\
table/mock_table.cc
\
test_util/fault_injection_test_env.cc
>>>>>>> 671d15cbd... Persistent Stats
:
persist stats history to disk (
#
5046)
BENCH_LIB_SOURCES
=
\
BENCH_LIB_SOURCES
=
\
tools/db_bench_tool.cc
\
<<<<<<<
HEAD
EXP_LIB_SOURCES
=
\
utilities/col_buf_decoder.cc
\
utilities/col_buf_encoder.cc
\
...
...
@@ -273,6 +307,12 @@ TEST_LIB_SOURCES = \
db/db_test_util.cc
\
util/testharness.cc
\
util/testutil.cc
\
=======
TEST_LIB_SOURCES
=
\
db/db_test_util.cc
\
test_util/testharness.cc
\
test_util/testutil.cc
\
>>>>>>>
671d15cbd... Persistent Stats: persist stats
history
to disk
(
#5046
)
utilities/cassandra/test_utils.cc
\
MAIN_SOURCES
=
\
...
...
@@ -370,12 +410,22 @@ MAIN_SOURCES = \
monitoring/histogram_test.cc
\
monitoring/iostats_context_test.cc
\
monitoring/statistics_test.cc
\
<<<<<<
< HEAD
monitoring/stats_dump_scheduler_test.cc
\
monitoring/stats_history_test.cc
\
options/options_settable_test.cc
options/options_test.cc
\
table/block_based_filter_block_test.cc
\
table/block_test.cc
\
=======
monitoring/stats_history_test.cc
\
options/options_test.cc
\
table/block_based/block_based_filter_block_test.cc
\
table/block_based/block_test.cc
\
table/block_based/data_block_hash_index_test.cc
\
table/block_based/full_filter_block_test.cc
\
table/block_based/partitioned_filter_block_test.cc
\
>>>>>>> 671d15cbd... Persistent Stats
:
persist stats history to disk (
#
5046)
table/cleanable_test.cc
\
table/cuckoo_table_builder_test.cc
\
table/cuckoo_table_reader_test.cc
\
...
...
@@ -396,9 +446,14 @@ MAIN_SOURCES = \
tools/ldb_cmd_test.cc
\
tools/reduce_levels_test.cc
\
tools/sst_dump_test.cc
\
<<<<<<<
HEAD
tools/trace_analyzer_test.cc
\
util/arena_test.cc
\
util/auto_roll_logger_test.cc
\
=======
tools/trace_analyzer_test.cc
\
trace_replay/block_cache_tracer_test.cc
\
>>>>>>> 671d15cbd... Persistent Stats
:
persist stats history to disk (
#
5046)
util/autovector_test.cc
\
util/bloom_test.cc
\
util/coding_test.cc
\
...
...
tools/db_bench_tool.cc
浏览文件 @
85df8fbb
...
...
@@ -1064,6 +1064,8 @@ DEFINE_uint64(stats_dump_period_sec, TERARKDB_NAMESPACE::Options().stats_dump_pe
DEFINE_uint64
(
stats_persist_period_sec
,
rocksdb
::
Options
().
stats_persist_period_sec
,
"Gap between persisting stats in seconds"
);
DEFINE_bool
(
persist_stats_to_disk
,
rocksdb
::
Options
().
persist_stats_to_disk
,
"whether to persist stats to disk"
);
DEFINE_uint64
(
stats_history_buffer_size
,
rocksdb
::
Options
().
stats_history_buffer_size
,
"Max number of stats snapshots to keep in memory"
);
...
...
@@ -3569,6 +3571,7 @@ class Benchmark {
static_cast
<
unsigned
int
>
(
FLAGS_stats_dump_period_sec
);
options
.
stats_persist_period_sec
=
static_cast
<
unsigned
int
>
(
FLAGS_stats_persist_period_sec
);
options
.
persist_stats_to_disk
=
FLAGS_persist_stats_to_disk
;
options
.
stats_history_buffer_size
=
static_cast
<
size_t
>
(
FLAGS_stats_history_buffer_size
);
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录