From 85df8fbb61e03620fd86923f405e3e660f48b817 Mon Sep 17 00:00:00 2001 From: Zhongyi Xie Date: Mon, 17 Jun 2019 15:17:43 -0700 Subject: [PATCH] [cherry-pick] --- CMakeLists.txt | 4 +- Makefile | 157 +++++++++ TARGETS | 8 +- db/db_impl.cc | 82 ++++- db/db_impl.h | 46 ++- db/db_impl_debug.cc | 4 +- db/db_impl_open.cc | 118 ++++++- db/db_options_test.cc | 2 + db/version_set.cc | 19 +- include/rocksdb/db.h | 10 +- include/rocksdb/options.h | 12 + include/rocksdb/stats_history.h | 4 +- {db => monitoring}/in_memory_stats_history.cc | 2 +- {db => monitoring}/in_memory_stats_history.h | 2 + monitoring/persistent_stats_history.cc | 171 ++++++++++ monitoring/persistent_stats_history.h | 83 +++++ monitoring/stats_history_test.cc | 297 +++++++++++++++++- options/db_options.cc | 5 +- options/db_options.h | 1 + options/options_helper.cc | 5 + options/options_settable_test.cc | 1 + options/options_test.cc | 2 + src.mk | 61 +++- tools/db_bench_tool.cc | 3 + 24 files changed, 1061 insertions(+), 38 deletions(-) rename {db => monitoring}/in_memory_stats_history.cc (97%) rename {db => monitoring}/in_memory_stats_history.h (93%) create mode 100644 monitoring/persistent_stats_history.cc create mode 100644 monitoring/persistent_stats_history.h diff --git a/CMakeLists.txt b/CMakeLists.txt index f391148c2..3139aa1e6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -558,7 +558,6 @@ set(SOURCES db/flush_scheduler.cc db/forward_iterator.cc db/internal_stats.cc - db/in_memory_stats_history.cc db/logs_with_prep_tracker.cc db/log_reader.cc db/log_writer.cc @@ -599,10 +598,12 @@ set(SOURCES memtable/write_buffer_manager.cc monitoring/histogram.cc monitoring/histogram_windowing.cc + monitoring/in_memory_stats_history.cc monitoring/instrumented_mutex.cc monitoring/iostats_context.cc monitoring/perf_context.cc monitoring/perf_level.cc + monitoring/persistent_stats_history.cc monitoring/statistics.cc monitoring/stats_dump_scheduler.cc monitoring/thread_status_impl.cc @@ -1005,6 +1006,7 @@ if(WITH_TESTS) monitoring/histogram_test.cc monitoring/iostats_context_test.cc monitoring/statistics_test.cc + monitoring/stats_history_test.cc options/options_settable_test.cc options/options_test.cc table/block_based_filter_block_test.cc diff --git a/Makefile b/Makefile index ad2a977b2..586550c41 100644 --- a/Makefile +++ b/Makefile @@ -518,10 +518,160 @@ ALL_SOURCES += $(TEST_MAIN_SOURCES) $(TOOL_MAIN_SOURCES) $(BENCH_MAIN_SOURCES) TESTS = $(patsubst %.cc, %, $(notdir $(TEST_MAIN_SOURCES))) TESTS += $(patsubst %.c, %, $(notdir $(TEST_MAIN_SOURCES_C))) +<<<<<<< HEAD ifeq ($(USE_FOLLY_DISTRIBUTED_MUTEX),1) TESTS += folly_synchronization_distributed_mutex_test ALL_SOURCES += third-party/folly/folly/synchronization/test/DistributedMutexTest.cc endif +======= +TESTS = \ + db_basic_test \ + db_encryption_test \ + db_test2 \ + external_sst_file_basic_test \ + auto_roll_logger_test \ + bloom_test \ + dynamic_bloom_test \ + c_test \ + checkpoint_test \ + crc32c_test \ + coding_test \ + inlineskiplist_test \ + env_basic_test \ + env_test \ + hash_test \ + thread_local_test \ + rate_limiter_test \ + perf_context_test \ + iostats_context_test \ + db_wal_test \ + db_block_cache_test \ + db_test \ + db_blob_index_test \ + db_bloom_filter_test \ + db_iter_test \ + db_iter_stress_test \ + db_log_iter_test \ + db_compaction_filter_test \ + db_compaction_test \ + db_dynamic_level_test \ + db_flush_test \ + db_inplace_update_test \ + db_iterator_test \ + db_memtable_test \ + db_merge_operator_test \ + db_options_test \ + db_range_del_test \ + db_secondary_test \ + db_sst_test \ + db_tailing_iter_test \ + db_io_failure_test \ + db_properties_test \ + db_table_properties_test \ + db_statistics_test \ + db_write_test \ + error_handler_test \ + autovector_test \ + blob_db_test \ + cleanable_test \ + column_family_test \ + table_properties_collector_test \ + arena_test \ + block_test \ + data_block_hash_index_test \ + cache_test \ + corruption_test \ + slice_transform_test \ + dbformat_test \ + fault_injection_test \ + filelock_test \ + filename_test \ + file_reader_writer_test \ + block_based_filter_block_test \ + full_filter_block_test \ + partitioned_filter_block_test \ + hash_table_test \ + histogram_test \ + log_test \ + manual_compaction_test \ + mock_env_test \ + memtable_list_test \ + merge_helper_test \ + memory_test \ + merge_test \ + merger_test \ + util_merge_operators_test \ + options_file_test \ + reduce_levels_test \ + plain_table_db_test \ + comparator_db_test \ + external_sst_file_test \ + prefix_test \ + skiplist_test \ + write_buffer_manager_test \ + stringappend_test \ + cassandra_format_test \ + cassandra_functional_test \ + cassandra_row_merge_test \ + cassandra_serialize_test \ + ttl_test \ + backupable_db_test \ + sim_cache_test \ + version_edit_test \ + version_set_test \ + compaction_picker_test \ + version_builder_test \ + file_indexer_test \ + write_batch_test \ + write_batch_with_index_test \ + write_controller_test\ + deletefile_test \ + obsolete_files_test \ + table_test \ + delete_scheduler_test \ + options_test \ + options_settable_test \ + options_util_test \ + event_logger_test \ + timer_queue_test \ + cuckoo_table_builder_test \ + cuckoo_table_reader_test \ + cuckoo_table_db_test \ + flush_job_test \ + wal_manager_test \ + listener_test \ + compaction_iterator_test \ + compaction_job_test \ + thread_list_test \ + sst_dump_test \ + compact_files_test \ + optimistic_transaction_test \ + write_callback_test \ + heap_test \ + compact_on_deletion_collector_test \ + compaction_job_stats_test \ + option_change_migration_test \ + transaction_test \ + ldb_cmd_test \ + persistent_cache_test \ + statistics_test \ + stats_history_test \ + lru_cache_test \ + object_registry_test \ + repair_test \ + env_timed_test \ + write_prepared_transaction_test \ + write_unprepared_transaction_test \ + db_universal_compaction_test \ + trace_analyzer_test \ + repeatable_thread_test \ + range_tombstone_fragmenter_test \ + range_del_aggregator_test \ + sst_file_reader_test \ + db_secondary_test \ + block_cache_tracer_test \ + block_cache_trace_analyzer_test \ +>>>>>>> 671d15cbd... Persistent Stats: persist stats history to disk (#5046) PARALLEL_TEST = \ backupable_db_test \ @@ -1781,7 +1931,14 @@ blob_db_test: $(OBJ_DIR)/utilities/blob_db/blob_db_test.o $(TEST_LIBRARY) $(LIBR repeatable_thread_test: $(OBJ_DIR)/util/repeatable_thread_test.o $(TEST_LIBRARY) $(LIBRARY) $(AM_LINK) +<<<<<<< HEAD range_tombstone_fragmenter_test: $(OBJ_DIR)/db/range_tombstone_fragmenter_test.o $(TEST_LIBRARY) $(LIBRARY) +======= +stats_history_test: monitoring/stats_history_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS) + $(AM_LINK) + +lru_cache_test: cache/lru_cache_test.o $(LIBOBJECTS) $(TESTHARNESS) +>>>>>>> 671d15cbd... Persistent Stats: persist stats history to disk (#5046) $(AM_LINK) sst_file_reader_test: $(OBJ_DIR)/table/sst_file_reader_test.o $(TEST_LIBRARY) $(LIBRARY) diff --git a/TARGETS b/TARGETS index f379438dd..de5200940 100644 --- a/TARGETS +++ b/TARGETS @@ -113,7 +113,6 @@ cpp_library( "db/flush_job.cc", "db/flush_scheduler.cc", "db/forward_iterator.cc", - "db/in_memory_stats_history.cc", "db/internal_stats.cc", "db/log_reader.cc", "db/log_writer.cc", @@ -154,10 +153,12 @@ cpp_library( "memtable/write_buffer_manager.cc", "monitoring/histogram.cc", "monitoring/histogram_windowing.cc", + "monitoring/in_memory_stats_history.cc", "monitoring/instrumented_mutex.cc", "monitoring/iostats_context.cc", "monitoring/perf_context.cc", "monitoring/perf_level.cc", + "monitoring/persistent_stats_history.cc", "monitoring/statistics.cc", "monitoring/stats_dump_scheduler.cc", "monitoring/thread_status_impl.cc", @@ -1000,6 +1001,11 @@ ROCKS_TESTS = [ [], [], ], + [ + "stats_history_test", + "monitoring/stats_history_test.cc", + "serial", + ], [ "stringappend_test", "utilities/merge_operators/string_append/stringappend_test.cc", diff --git a/db/db_impl.cc b/db/db_impl.cc index 3197f5cab..533949282 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -37,7 +37,6 @@ #include "db/external_sst_file_ingestion_job.h" #include "db/flush_job.h" #include "db/forward_iterator.h" -#include "db/in_memory_stats_history.h" #include "db/job_context.h" #include "db/log_reader.h" #include "db/log_writer.h" @@ -56,10 +55,12 @@ #include "db/write_callback.h" #include "memtable/hash_linklist_rep.h" #include "memtable/hash_skiplist_rep.h" +#include "monitoring/in_memory_stats_history.h" #include "monitoring/iostats_context_imp.h" #include "monitoring/perf_context_imp.h" #include "trace_replay/block_cache_tracer.h" #include "monitoring/stats_dump_scheduler.h" +#include "monitoring/persistent_stats_history.h" #include "monitoring/thread_status_updater.h" #include "monitoring/thread_status_util.h" #include "options/cf_options.h" @@ -215,6 +216,7 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname, log_dir_synced_(false), log_empty_(true), default_cf_handle_(nullptr), + persist_stats_cf_handle_(nullptr), log_sync_cv_(&mutex_), total_log_size_(0), max_total_in_memory_state_(0), @@ -625,10 +627,17 @@ Status DBImpl::CloseHelper() { } memtable_info_queue_.clear(); - if (default_cf_handle_ != nullptr) { + if (default_cf_handle_ != nullptr || persist_stats_cf_handle_ != nullptr) { // we need to delete handle outside of lock because it does its own locking mutex_.Unlock(); - delete default_cf_handle_; + if (default_cf_handle_) { + delete default_cf_handle_; + default_cf_handle_ = nullptr; + } + if (persist_stats_cf_handle_) { + delete persist_stats_cf_handle_; + persist_stats_cf_handle_ = nullptr; + } mutex_.Lock(); } @@ -780,7 +789,7 @@ void DBImpl::StartStatsDumpScheduler() { } // esitmate the total size of stats_history_ -size_t DBImpl::EstiamteStatsHistorySize() const { +size_t DBImpl::EstimateInMemoryStatsHistorySize() const { size_t size_total = sizeof(std::map>); if (stats_history_.size() == 0) return size_total; @@ -814,12 +823,40 @@ void DBImpl::PersistStats() { stats_history_size_limit = mutable_db_options_.stats_history_buffer_size; } - // TODO(Zhongyi): also persist immutable_db_options_.statistics - { - std::map stats_map; - if (!statistics->getTickerMap(&stats_map)) { - return; + std::map stats_map; + if (!statistics->getTickerMap(&stats_map)) { + return; + } + + if (immutable_db_options_.persist_stats_to_disk) { + WriteBatch batch; + if (stats_slice_initialized_) { + for (const auto& stat : stats_map) { + char key[100]; + int length = + EncodePersistentStatsKey(now_seconds, stat.first, 100, key); + // calculate the delta from last time + if (stats_slice_.find(stat.first) != stats_slice_.end()) { + uint64_t delta = stat.second - stats_slice_[stat.first]; + batch.Put(persist_stats_cf_handle_, Slice(key, std::min(100, length)), + ToString(delta)); + } + } } + stats_slice_initialized_ = true; + std::swap(stats_slice_, stats_map); + WriteOptions wo; + wo.low_pri = true; + wo.no_slowdown = true; + wo.sync = false; + Status s = Write(wo, &batch); + if (!s.ok()) { + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "Writing to persistent stats CF failed -- %s\n", + s.ToString().c_str()); + } + // TODO(Zhongyi): add purging for persisted data + } else { InstrumentedMutexLock l(&stats_history_mutex_); // calculate the delta from last time if (stats_slice_initialized_) { @@ -829,17 +866,19 @@ void DBImpl::PersistStats() { stats_delta[stat.first] = stat.second - stats_slice_[stat.first]; } } - stats_history_[now_micros] = stats_delta; + stats_history_[now_seconds] = stats_delta; } stats_slice_initialized_ = true; std::swap(stats_slice_, stats_map); TEST_SYNC_POINT("DBImpl::PersistStats:StatsCopied"); // delete older stats snapshots to control memory consumption - bool purge_needed = EstiamteStatsHistorySize() > stats_history_size_limit; + bool purge_needed = + EstimateInMemoryStatsHistorySize() > stats_history_size_limit; while (purge_needed && !stats_history_.empty()) { stats_history_.erase(stats_history_.begin()); - purge_needed = EstiamteStatsHistorySize() > stats_history_size_limit; + purge_needed = + EstimateInMemoryStatsHistorySize() > stats_history_size_limit; } } @@ -874,8 +913,13 @@ Status DBImpl::GetStatsHistory( if (!stats_iterator) { return Status::InvalidArgument("stats_iterator not preallocated."); } - stats_iterator->reset( - new InMemoryStatsHistoryIterator(start_time, end_time, this)); + if (immutable_db_options_.persist_stats_to_disk) { + stats_iterator->reset( + new PersistentStatsHistoryIterator(start_time, end_time, this)); + } else { + stats_iterator->reset( + new InMemoryStatsHistoryIterator(start_time, end_time, this)); + } return (*stats_iterator)->status(); } void DBImpl::ScheduleGCTTL() { @@ -1131,14 +1175,14 @@ Status DBImpl::SetDBOptions( stats_dump_scheduler_->Unregister(this); mutex_.Lock(); } - if (new_options.stats_dump_period_sec > 0 || - new_options.stats_persist_period_sec > 0) { + } + if (new_options.stats_persist_period_sec != + mutable_db_options_.stats_persist_period_sec) { mutex_.Unlock(); stats_dump_scheduler_->Register(this, new_options.stats_dump_period_sec, new_options.stats_persist_period_sec); mutex_.Lock(); - } } write_controller_.set_max_delayed_write_rate( new_options.delayed_write_rate); @@ -1561,6 +1605,10 @@ ColumnFamilyHandle* DBImpl::DefaultColumnFamily() const { return default_cf_handle_; } +ColumnFamilyHandle* DBImpl::PersistentStatsColumnFamily() const { + return persist_stats_cf_handle_; +} + Status DBImpl::Get(const ReadOptions& read_options, ColumnFamilyHandle* column_family, const Slice& key, LazyBuffer* value) { diff --git a/db/db_impl.h b/db/db_impl.h index 007409cee..3bcb52f62 100644 --- a/db/db_impl.h +++ b/db/db_impl.h @@ -68,12 +68,12 @@ class Arena; class ArenaWrappedDBIter; class InMemoryStatsHistoryIterator; class MemTable; - class PersistentStatsHistoryIterator; class StatsDumpScheduler; #ifndef NDEBUG class StatsDumpTestScheduler; #endif // !NDEBUG + class TableCache; class Version; class VersionEdit; @@ -255,6 +255,17 @@ class DBImpl : public DB { virtual bool SetPreserveDeletesSequenceNumber(SequenceNumber seqnum) override; + virtual Status GetDbIdentity(std::string& identity) const override; + + ColumnFamilyHandle* DefaultColumnFamily() const override; + + ColumnFamilyHandle* PersistentStatsColumnFamily() const; + + virtual Status Close() override; + + Status GetStatsHistory( + uint64_t start_time, uint64_t end_time, + std::unique_ptr* stats_iterator) override; #ifndef ROCKSDB_LITE using DB::ResetStats; virtual Status ResetStats() override; @@ -782,9 +793,19 @@ class DBImpl : public DB { VersionSet* TEST_GetVersionSet() const { return versions_.get(); } +<<<<<<< HEAD:db/db_impl.h #ifndef ROCKSDB_LITE StatsDumpTestScheduler* TEST_GetStatsDumpScheduler() const; #endif // !ROCKSDB_LITE +======= + int TEST_BGCompactionsAllowed() const; + int TEST_BGFlushesAllowed() const; + size_t TEST_GetWalPreallocateBlockSize(uint64_t write_buffer_size) const; + void TEST_WaitForDumpStatsRun(std::function callback) const; + void TEST_WaitForPersistStatsRun(std::function callback) const; + bool TEST_IsPersistentStatsEnabled() const; + size_t TEST_EstimateInMemoryStatsHistorySize() const; +>>>>>>> 671d15cbd... Persistent Stats: persist stats history to disk (#5046):db/db_impl/db_impl.h #endif // NDEBUG protected: @@ -910,6 +931,9 @@ class DBImpl : public DB { friend class CompactedDBImpl; friend class DBTest_ConcurrentFlushWAL_Test; friend class DBTest_MixedSlowdownOptionsStop_Test; + friend class DBCompactionTest_CompactBottomLevelFilesWithDeletions_Test; + friend class DBCompactionTest_CompactionDuringShutdown_Test; + friend class StatsHistoryTest_PersistentStatsCreateColumnFamilies_Test; #ifndef NDEBUG friend class DBTest2_ReadCallbackTest_Test; friend class WriteCallbackTest_WriteWithCallbackTest_Test; @@ -943,6 +967,21 @@ class DBImpl : public DB { bool read_only = false, bool error_if_log_file_exist = false, bool error_if_data_exists_in_logs = false); + // Initialize the built-in column family for persistent stats. Depending on + // whether on-disk persistent stats have been enabled before, it may either + // create a new column family and column family handle or just a column family + // handle. + // Required: DB mutex held + Status InitPersistStatsColumnFamily(); + + // Persistent Stats column family has two format version key which are used + // for compatibility check. Write format version if it's created for the + // first time, read format version and check compatibility if recovering + // from disk. This function requires DB mutex held at entrance but may + // release and re-acquire DB mutex in the process. + // Required: DB mutex held + Status PersistentStatsProcessFormatVersion(); + Status ResumeImpl(); void MaybeIgnoreError(Status* s) const; @@ -1198,7 +1237,7 @@ class DBImpl : public DB { void PrintStatistics(); - size_t EstiamteStatsHistorySize() const; + size_t EstimateInMemoryStatsHistorySize() const; // persist stats to column family "_persistent_stats" void PersistStats(); @@ -1337,6 +1376,9 @@ class DBImpl : public DB { // true for some prefix of logs_ bool getting_synced = false; }; + + ColumnFamilyHandleImpl* persist_stats_cf_handle_; + bool persistent_stats_cfd_exists_ = true; // Without two_write_queues, read and writes to alive_log_files_ are // protected by mutex_. However since back() is never popped, and push_back() // is done only from write_thread_, the same thread can access the item diff --git a/db/db_impl_debug.cc b/db/db_impl_debug.cc index ee6f0cd77..9568e2564 100644 --- a/db/db_impl_debug.cc +++ b/db/db_impl_debug.cc @@ -265,8 +265,8 @@ StatsDumpTestScheduler* DBImpl::TEST_GetStatsDumpScheduler() const { } #endif // !ROCKSDB_LITE -size_t DBImpl::TEST_EstiamteStatsHistorySize() const { - return EstiamteStatsHistorySize(); +size_t DBImpl::TEST_EstimateInMemoryStatsHistorySize() const { + return EstimateInMemoryStatsHistorySize(); } } // namespace TERARKDB_NAMESPACE #endif // NDEBUG diff --git a/db/db_impl_open.cc b/db/db_impl_open.cc index 94206f5c5..08fdf9f71 100644 --- a/db/db_impl_open.cc +++ b/db/db_impl_open.cc @@ -16,6 +16,7 @@ #include "db/builder.h" #include "db/error_handler.h" #include "db/map_builder.h" +#include "monitoring/persistent_stats_history.h" #include "options/options_helper.h" #include "rocksdb/wal_filter.h" #include "table/block_based_table_factory.h" @@ -23,6 +24,7 @@ #include "util/rate_limiter.h" #include "util/sst_file_manager_impl.h" #include "util/sync_point.h" +#include "util/string_util.h" #if !defined(_MSC_VER) && !defined(__APPLE__) #include #include @@ -405,6 +407,7 @@ Status DBImpl::Recover( } Status s = versions_->Recover(column_families, read_only); + if (immutable_db_options_.paranoid_checks && s.ok()) { s = CheckConsistency(read_only); } @@ -416,6 +419,10 @@ Status DBImpl::Recover( } } } + // DB mutex is already held + if (s.ok() && immutable_db_options_.persist_stats_to_disk) { + s = InitPersistStatsColumnFamily(); + } // Initial max_total_in_memory_state_ before recovery logs. Log recovery // may check this value to decide whether to flush. @@ -431,6 +438,8 @@ Status DBImpl::Recover( default_cf_handle_ = new ColumnFamilyHandleImpl( versions_->GetColumnFamilySet()->GetDefault(), this, &mutex_); default_cf_internal_stats_ = default_cf_handle_->cfd()->internal_stats(); + // TODO(Zhongyi): handle single_column_family_mode_ when + // persistent_stats is enabled single_column_family_mode_ = versions_->GetColumnFamilySet()->NumberOfColumnFamilies() == 1; @@ -538,6 +547,98 @@ Status DBImpl::Recover( return s; } +Status DBImpl::PersistentStatsProcessFormatVersion() { + mutex_.AssertHeld(); + Status s; + // persist version when stats CF doesn't exist + bool should_persist_format_version = !persistent_stats_cfd_exists_; + mutex_.Unlock(); + if (persistent_stats_cfd_exists_) { + // Check persistent stats format version compatibility. Drop and recreate + // persistent stats CF if format version is incompatible + uint64_t format_version_recovered = 0; + Status s_format = DecodePersistentStatsVersionNumber( + this, StatsVersionKeyType::kFormatVersion, &format_version_recovered); + uint64_t compatible_version_recovered = 0; + Status s_compatible = DecodePersistentStatsVersionNumber( + this, StatsVersionKeyType::kCompatibleVersion, + &compatible_version_recovered); + // abort reading from existing stats CF if any of following is true: + // 1. failed to read format version or compatible version from disk + // 2. sst's format version is greater than current format version, meaning + // this sst is encoded with a newer RocksDB release, and current compatible + // version is below the sst's compatible version + if (!s_format.ok() || !s_compatible.ok() || + (kStatsCFCurrentFormatVersion < format_version_recovered && + kStatsCFCompatibleFormatVersion < compatible_version_recovered)) { + if (!s_format.ok() || !s_compatible.ok()) { + ROCKS_LOG_INFO( + immutable_db_options_.info_log, + "Reading persistent stats version key failed. Format key: %s, " + "compatible key: %s", + s_format.ToString().c_str(), s_compatible.ToString().c_str()); + } else { + ROCKS_LOG_INFO( + immutable_db_options_.info_log, + "Disable persistent stats due to corrupted or incompatible format " + "version\n"); + } + DropColumnFamily(persist_stats_cf_handle_); + DestroyColumnFamilyHandle(persist_stats_cf_handle_); + ColumnFamilyHandle* handle = nullptr; + ColumnFamilyOptions cfo; + OptimizeForPersistentStats(&cfo); + s = CreateColumnFamily(cfo, kPersistentStatsColumnFamilyName, &handle); + persist_stats_cf_handle_ = static_cast(handle); + // should also persist version here because old stats CF is discarded + should_persist_format_version = true; + } + } + if (s.ok() && should_persist_format_version) { + // Persistent stats CF being created for the first time, need to write + // format version key + WriteBatch batch; + batch.Put(persist_stats_cf_handle_, kFormatVersionKeyString, + ToString(kStatsCFCurrentFormatVersion)); + batch.Put(persist_stats_cf_handle_, kCompatibleVersionKeyString, + ToString(kStatsCFCompatibleFormatVersion)); + WriteOptions wo; + wo.low_pri = true; + wo.no_slowdown = true; + wo.sync = false; + s = Write(wo, &batch); + } + mutex_.Lock(); + return s; +} + +Status DBImpl::InitPersistStatsColumnFamily() { + mutex_.AssertHeld(); + assert(!persist_stats_cf_handle_); + ColumnFamilyData* persistent_stats_cfd = + versions_->GetColumnFamilySet()->GetColumnFamily( + kPersistentStatsColumnFamilyName); + persistent_stats_cfd_exists_ = persistent_stats_cfd != nullptr; + + Status s; + if (persistent_stats_cfd != nullptr) { + // We are recovering from a DB which already contains persistent stats CF, + // the CF is already created in VersionSet::ApplyOneVersionEdit, but + // column family handle was not. Need to explicitly create handle here. + persist_stats_cf_handle_ = + new ColumnFamilyHandleImpl(persistent_stats_cfd, this, &mutex_); + } else { + mutex_.Unlock(); + ColumnFamilyHandle* handle = nullptr; + ColumnFamilyOptions cfo; + OptimizeForPersistentStats(&cfo); + s = CreateColumnFamily(cfo, kPersistentStatsColumnFamilyName, &handle); + persist_stats_cf_handle_ = static_cast(handle); + mutex_.Lock(); + } + return s; +} + // REQUIRES: log_numbers are sorted in ascending order Status DBImpl::RecoverLogFiles(const std::vector& log_numbers, SequenceNumber* next_sequence, bool read_only) { @@ -1119,12 +1220,23 @@ Status DB::Open(const Options& options, const std::string& dbname, DB** dbptr) { std::vector column_families; column_families.push_back( ColumnFamilyDescriptor(kDefaultColumnFamilyName, cf_options)); + if (db_options.persist_stats_to_disk) { + column_families.push_back( + ColumnFamilyDescriptor(kPersistentStatsColumnFamilyName, cf_options)); + } std::vector handles; Status s = DB::Open(db_options, dbname, column_families, &handles, dbptr); if (s.ok()) { - assert(handles.size() == 1); + if (db_options.persist_stats_to_disk) { + assert(handles.size() == 2); + } else { + assert(handles.size() == 1); + } // i can delete the handle since DBImpl is always holding a reference to // default column family + if (db_options.persist_stats_to_disk && handles[1] != nullptr) { + delete handles[1]; + } delete handles[0]; } return s; @@ -1298,6 +1410,10 @@ Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname, s = impl->directories_.GetDbDir()->Fsync(); } } + if (s.ok() && impl->immutable_db_options_.persist_stats_to_disk) { + // try to read format version but no need to fail Open() even if it fails + s = impl->PersistentStatsProcessFormatVersion(); + } if (s.ok()) { for (auto cfd : *impl->versions_->GetColumnFamilySet()) { diff --git a/db/db_options_test.cc b/db/db_options_test.cc index 7c466bd39..f7824e17c 100644 --- a/db/db_options_test.cc +++ b/db/db_options_test.cc @@ -537,6 +537,7 @@ TEST_F(DBOptionsTest, SetStatsDumpPeriodSec) { Close(); } +<<<<<<< HEAD TEST_F(DBOptionsTest, RunStatsDumpPeriodSec) { Options options; options.create_if_missing = true; @@ -681,6 +682,7 @@ TEST_F(DBOptionsTest, SetOptionsStatsPersistPeriodSec) { ASSERT_EQ(12345, dbfull()->GetDBOptions().stats_persist_period_sec); } +<<<<<<< HEAD TEST_F(DBOptionsTest, GetStatsHistory) { Options options; options.create_if_missing = true; diff --git a/db/version_set.cc b/db/version_set.cc index 5672cd8f7..6e34030ee 100644 --- a/db/version_set.cc +++ b/db/version_set.cc @@ -17,6 +17,8 @@ #include #include +#include +#include #include #include #include @@ -35,6 +37,7 @@ #include "db/version_builder.h" #include "monitoring/file_read_sample.h" #include "monitoring/perf_context_imp.h" +#include "monitoring/persistent_stats_history.h" #include "rocksdb/env.h" #include "rocksdb/merge_operator.h" #include "rocksdb/write_buffer_manager.h" @@ -3510,11 +3513,23 @@ Status VersionSet::ApplyOneVersionEdit( edit.column_family_name_); } auto cf_options = name_to_options.find(edit.column_family_name_); - if (cf_options == name_to_options.end()) { + // implicitly add persistent_stats column family without requiring user + // to specify + bool is_persistent_stats_column_family = + edit.column_family_name_.compare(kPersistentStatsColumnFamilyName) == 0; + if (cf_options == name_to_options.end() && + !is_persistent_stats_column_family) { column_families_not_found.insert( {edit.column_family_, edit.column_family_name_}); } else { - cfd = CreateColumnFamily(cf_options->second, &edit); + // recover persistent_stats CF from a DB that already contains it + if (is_persistent_stats_column_family) { + ColumnFamilyOptions cfo; + OptimizeForPersistentStats(&cfo); + cfd = CreateColumnFamily(cfo, &edit); + } else { + cfd = CreateColumnFamily(cf_options->second, &edit); + } cfd->set_initialized(); builders.insert( {edit.column_family_, new BaseReferencedVersionBuilder(cfd)}); diff --git a/include/rocksdb/db.h b/include/rocksdb/db.h index 9bc3f8430..fc8b5e6d1 100644 --- a/include/rocksdb/db.h +++ b/include/rocksdb/db.h @@ -72,6 +72,7 @@ class TraceWriter; using std::unique_ptr; extern const std::string kDefaultColumnFamilyName; +extern const std::string kPersistentStatsColumnFamilyName; struct ColumnFamilyDescriptor { std::string name; ColumnFamilyOptions options; @@ -1275,10 +1276,11 @@ class DB { // Needed for StackableDB virtual DB* GetRootDB() { return this; } - // Given a time window, return an iterator for accessing stats history - // User is responsible for deleting StatsHistoryIterator after use - virtual Status GetStatsHistory(uint64_t /*start_time*/, - uint64_t /*end_time*/, + // Given a window [start_time, end_time), setup a StatsHistoryIterator + // to access stats history. Note the start_time and end_time are epoch + // time measured in seconds, and end_time is an exclusive bound. + virtual Status GetStatsHistory( + uint64_t /*start_time*/, uint64_t /*end_time*/, std::unique_ptr* /*stats_iterator*/) { return Status::NotSupported("GetStatsHistory() is not implemented."); } diff --git a/include/rocksdb/options.h b/include/rocksdb/options.h index ff00e9bd0..80bedf7d1 100644 --- a/include/rocksdb/options.h +++ b/include/rocksdb/options.h @@ -737,6 +737,18 @@ struct DBOptions { // Default: 600 unsigned int stats_persist_period_sec = 600; + // If true, automatically persist stats to a hidden column family (column + // family name: ___rocksdb_stats_history___) every + // stats_persist_period_sec seconds; otherwise, write to an in-memory + // struct. User can query through `GetStatsHistory` API. + // If user attempts to create a column family with the same name on a DB + // which have previously set persist_stats_to_disk to true, the column family + // creation will fail, but the hidden column family will survive, as well as + // the previously persisted statistics. + // When peristing stats to disk, the stat name will be limited at 100 bytes. + // Default: false + bool persist_stats_to_disk = false; + // if not zero, periodically take stats snapshots and store in memory, the // memory size for stats snapshots is capped at stats_history_buffer_size // Default: 1MB diff --git a/include/rocksdb/stats_history.h b/include/rocksdb/stats_history.h index 40ea51d1f..8b73deb44 100644 --- a/include/rocksdb/stats_history.h +++ b/include/rocksdb/stats_history.h @@ -31,10 +31,12 @@ class StatsHistoryIterator { // REQUIRES: Valid() virtual void Next() = 0; - // Return the time stamp (in microseconds) when stats history is recorded. + // Return the time stamp (in seconds) when stats history is recorded. // REQUIRES: Valid() virtual uint64_t GetStatsTime() const = 0; + virtual int GetFormatVersion() const { return -1; } + // Return the current stats history as an std::map which specifies the // mapping from stats name to stats value . The underlying storage // for the returned map is valid only until the next modification of diff --git a/db/in_memory_stats_history.cc b/monitoring/in_memory_stats_history.cc similarity index 97% rename from db/in_memory_stats_history.cc rename to monitoring/in_memory_stats_history.cc index 39355cfbe..44f087173 100644 --- a/db/in_memory_stats_history.cc +++ b/monitoring/in_memory_stats_history.cc @@ -7,7 +7,7 @@ // found in the LICENSE file. See the AUTHORS file for names of contributors. #include "db/db_impl.h" -#include "db/in_memory_stats_history.h" +#include "monitoring/in_memory_stats_history.h" namespace rocksdb { diff --git a/db/in_memory_stats_history.h b/monitoring/in_memory_stats_history.h similarity index 93% rename from db/in_memory_stats_history.h rename to monitoring/in_memory_stats_history.h index 4b52e23ff..edcb9b4a5 100644 --- a/db/in_memory_stats_history.h +++ b/monitoring/in_memory_stats_history.h @@ -14,6 +14,8 @@ namespace rocksdb { class InMemoryStatsHistoryIterator final : public StatsHistoryIterator { public: + // Setup InMemoryStatsHistoryIterator to return stats snapshots between + // seconds timestamps [start_time, end_time) InMemoryStatsHistoryIterator(uint64_t start_time, uint64_t end_time, DBImpl* db_impl) : start_time_(start_time), diff --git a/monitoring/persistent_stats_history.cc b/monitoring/persistent_stats_history.cc new file mode 100644 index 000000000..c1704f567 --- /dev/null +++ b/monitoring/persistent_stats_history.cc @@ -0,0 +1,171 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "monitoring/persistent_stats_history.h" + +#include +#include +#include +#include "db/db_impl/db_impl.h" +#include "port/likely.h" +#include "util/string_util.h" + +namespace rocksdb { +// 10 digit seconds timestamp => [Sep 9, 2001 ~ Nov 20, 2286] +const int kNowSecondsStringLength = 10; +const std::string kFormatVersionKeyString = + "__persistent_stats_format_version__"; +const std::string kCompatibleVersionKeyString = + "__persistent_stats_compatible_version__"; +// Every release maintains two versions numbers for persistents stats: Current +// format version and compatible format version. Current format version +// designates what type of encoding will be used when writing to stats CF; +// compatible format version designates the minimum format version that +// can decode the stats CF encoded using the current format version. +const uint64_t kStatsCFCurrentFormatVersion = 1; +const uint64_t kStatsCFCompatibleFormatVersion = 1; + +Status DecodePersistentStatsVersionNumber(DBImpl* db, StatsVersionKeyType type, + uint64_t* version_number) { + if (type >= StatsVersionKeyType::kKeyTypeMax) { + return Status::InvalidArgument("Invalid stats version key type provided"); + } + std::string key; + if (type == StatsVersionKeyType::kFormatVersion) { + key = kFormatVersionKeyString; + } else if (type == StatsVersionKeyType::kCompatibleVersion) { + key = kCompatibleVersionKeyString; + } + ReadOptions options; + options.verify_checksums = true; + std::string result; + Status s = db->Get(options, db->PersistentStatsColumnFamily(), key, &result); + if (!s.ok() || result.empty()) { + return Status::NotFound("Persistent stats version key " + key + + " not found."); + } + + // read version_number but do nothing in current version + *version_number = ParseUint64(result); + return Status::OK(); +} + +int EncodePersistentStatsKey(uint64_t now_seconds, const std::string& key, + int size, char* buf) { + char timestamp[kNowSecondsStringLength + 1]; + // make time stamp string equal in length to allow sorting by time + snprintf(timestamp, sizeof(timestamp), "%010d", + static_cast(now_seconds)); + timestamp[kNowSecondsStringLength] = '\0'; + return snprintf(buf, size, "%s#%s", timestamp, key.c_str()); +} + +void OptimizeForPersistentStats(ColumnFamilyOptions* cfo) { + cfo->write_buffer_size = 2 << 20; + cfo->target_file_size_base = 2 * 1048576; + cfo->max_bytes_for_level_base = 10 * 1048576; + cfo->snap_refresh_nanos = 0; + cfo->soft_pending_compaction_bytes_limit = 256 * 1048576; + cfo->hard_pending_compaction_bytes_limit = 1073741824ul; + cfo->compression = kNoCompression; +} + +PersistentStatsHistoryIterator::~PersistentStatsHistoryIterator() {} + +bool PersistentStatsHistoryIterator::Valid() const { return valid_; } + +Status PersistentStatsHistoryIterator::status() const { return status_; } + +void PersistentStatsHistoryIterator::Next() { + // increment start_time by 1 to avoid infinite loop + AdvanceIteratorByTime(GetStatsTime() + 1, end_time_); +} + +uint64_t PersistentStatsHistoryIterator::GetStatsTime() const { return time_; } + +const std::map& +PersistentStatsHistoryIterator::GetStatsMap() const { + return stats_map_; +} + +std::pair parseKey(const Slice& key, + uint64_t start_time) { + std::pair result; + std::string key_str = key.ToString(); + std::string::size_type pos = key_str.find("#"); + // TODO(Zhongyi): add counters to track parse failures? + if (pos == std::string::npos) { + result.first = port::kMaxUint64; + result.second.clear(); + } else { + uint64_t parsed_time = ParseUint64(key_str.substr(0, pos)); + // skip entries with timestamp smaller than start_time + if (parsed_time < start_time) { + result.first = port::kMaxUint64; + result.second = ""; + } else { + result.first = parsed_time; + std::string key_resize = key_str.substr(pos + 1); + result.second = key_resize; + } + } + return result; +} + +// advance the iterator to the next time between [start_time, end_time) +// if success, update time_ and stats_map_ with new_time and stats_map +void PersistentStatsHistoryIterator::AdvanceIteratorByTime(uint64_t start_time, + uint64_t end_time) { + // try to find next entry in stats_history_ map + if (db_impl_ != nullptr) { + ReadOptions ro; + Iterator* iter = + db_impl_->NewIterator(ro, db_impl_->PersistentStatsColumnFamily()); + + char timestamp[kNowSecondsStringLength + 1]; + snprintf(timestamp, sizeof(timestamp), "%010d", + static_cast(std::max(time_, start_time))); + timestamp[kNowSecondsStringLength] = '\0'; + + iter->Seek(timestamp); + // no more entries with timestamp >= start_time is found or version key + // is found to be incompatible + if (!iter->Valid()) { + valid_ = false; + delete iter; + return; + } + time_ = parseKey(iter->key(), start_time).first; + valid_ = true; + // check parsed time and invalid if it exceeds end_time + if (time_ > end_time) { + valid_ = false; + delete iter; + return; + } + // find all entries with timestamp equal to time_ + std::map new_stats_map; + std::pair kv; + for (; iter->Valid(); iter->Next()) { + kv = parseKey(iter->key(), start_time); + if (kv.first != time_) { + break; + } + if (kv.second.compare(kFormatVersionKeyString) == 0) { + continue; + } + new_stats_map[kv.second] = ParseUint64(iter->value().ToString()); + } + stats_map_.swap(new_stats_map); + delete iter; + } else { + valid_ = false; + } +} + +} // namespace rocksdb diff --git a/monitoring/persistent_stats_history.h b/monitoring/persistent_stats_history.h new file mode 100644 index 000000000..9a6885987 --- /dev/null +++ b/monitoring/persistent_stats_history.h @@ -0,0 +1,83 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include "db/db_impl/db_impl.h" +#include "rocksdb/stats_history.h" + +namespace rocksdb { + +extern const std::string kFormatVersionKeyString; +extern const std::string kCompatibleVersionKeyString; +extern const uint64_t kStatsCFCurrentFormatVersion; +extern const uint64_t kStatsCFCompatibleFormatVersion; + +enum StatsVersionKeyType : uint32_t { + kFormatVersion = 1, + kCompatibleVersion = 2, + kKeyTypeMax = 3 +}; + +// Read the version number from persitent stats cf depending on type provided +// stores the version number in `*version_number` +// returns Status::OK() on success, or other status code on failure +Status DecodePersistentStatsVersionNumber(DBImpl* db, StatsVersionKeyType type, + uint64_t* version_number); + +// Encode timestamp and stats key into buf +// Format: timestamp(10 digit) + '#' + key +// Total length of encoded key will be capped at 100 bytes +int EncodePersistentStatsKey(uint64_t timestamp, const std::string& key, + int size, char* buf); + +void OptimizeForPersistentStats(ColumnFamilyOptions* cfo); + +class PersistentStatsHistoryIterator final : public StatsHistoryIterator { + public: + PersistentStatsHistoryIterator(uint64_t start_time, uint64_t end_time, + DBImpl* db_impl) + : time_(0), + start_time_(start_time), + end_time_(end_time), + valid_(true), + db_impl_(db_impl) { + AdvanceIteratorByTime(start_time_, end_time_); + } + ~PersistentStatsHistoryIterator() override; + bool Valid() const override; + Status status() const override; + + void Next() override; + uint64_t GetStatsTime() const override; + + const std::map& GetStatsMap() const override; + + private: + // advance the iterator to the next stats history record with timestamp + // between [start_time, end_time) + void AdvanceIteratorByTime(uint64_t start_time, uint64_t end_time); + + // No copying allowed + PersistentStatsHistoryIterator(const PersistentStatsHistoryIterator&) = + delete; + void operator=(const PersistentStatsHistoryIterator&) = delete; + PersistentStatsHistoryIterator(PersistentStatsHistoryIterator&&) = delete; + PersistentStatsHistoryIterator& operator=(PersistentStatsHistoryIterator&&) = + delete; + + uint64_t time_; + uint64_t start_time_; + uint64_t end_time_; + std::map stats_map_; + Status status_; + bool valid_; + DBImpl* db_impl_; +}; + +} // namespace rocksdb diff --git a/monitoring/stats_history_test.cc b/monitoring/stats_history_test.cc index 13edd2ac8..db60a3f7d 100644 --- a/monitoring/stats_history_test.cc +++ b/monitoring/stats_history_test.cc @@ -6,7 +6,6 @@ // Copyright (c) 2011 The LevelDB Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. -#include "rocksdb/stats_history.h" #include #include @@ -16,12 +15,12 @@ #include "db/db_impl/db_impl.h" #include "db/db_test_util.h" #include "monitoring/persistent_stats_history.h" -#include "monitoring/stats_dump_scheduler.h" #include "options/options_helper.h" #include "port/stack_trace.h" #include "rocksdb/cache.h" #include "rocksdb/convenience.h" #include "rocksdb/rate_limiter.h" +#include "rocksdb/stats_history.h" #include "test_util/sync_point.h" #include "test_util/testutil.h" #include "util/random.h" @@ -86,6 +85,7 @@ TEST_F(StatsHistoryTest, RunStatsDumpPeriodSec) { // Test persistent stats background thread scheduling and cancelling TEST_F(StatsHistoryTest, StatsPersistScheduling) { +<<<<<<< HEAD constexpr int kPeriodSec = 5; Options options; options.create_if_missing = true; @@ -118,6 +118,39 @@ TEST_F(StatsHistoryTest, StatsPersistScheduling) { [&] { mock_env_->set_current_time(mock_time_sec); }); ASSERT_EQ(counter, old_val); +======= + Options options; + options.create_if_missing = true; + options.stats_persist_period_sec = 5; + std::unique_ptr mock_env; + mock_env.reset(new rocksdb::MockTimeEnv(env_)); + mock_env->set_current_time(0); // in seconds + options.env = mock_env.get(); + rocksdb::SyncPoint::GetInstance()->DisableProcessing(); + rocksdb::SyncPoint::GetInstance()->ClearAllCallBacks(); +#if defined(OS_MACOSX) && !defined(NDEBUG) + rocksdb::SyncPoint::GetInstance()->SetCallBack( + "InstrumentedCondVar::TimedWaitInternal", [&](void* arg) { + uint64_t time_us = *reinterpret_cast(arg); + if (time_us < mock_env->RealNowMicros()) { + *reinterpret_cast(arg) = mock_env->RealNowMicros() + 1000; + } + }); +#endif // OS_MACOSX && !NDEBUG + int counter = 0; + rocksdb::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::PersistStats:Entry", [&](void* /*arg*/) { counter++; }); + rocksdb::SyncPoint::GetInstance()->EnableProcessing(); + Reopen(options); + ASSERT_EQ(5, dbfull()->GetDBOptions().stats_persist_period_sec); + dbfull()->TEST_WaitForPersistStatsRun([&] { mock_env->set_current_time(5); }); + ASSERT_GE(counter, 1); + + // Test cacel job through SetOptions + ASSERT_TRUE(dbfull()->TEST_IsPersistentStatsEnabled()); + ASSERT_OK(dbfull()->SetDBOptions({{"stats_persist_period_sec", "0"}})); + ASSERT_FALSE(dbfull()->TEST_IsPersistentStatsEnabled()); +>>>>>>> 671d15cbd... Persistent Stats: persist stats history to disk (#5046) Close(); } @@ -126,6 +159,7 @@ TEST_F(StatsHistoryTest, PersistentStatsFreshInstall) { Options options; options.create_if_missing = true; options.stats_persist_period_sec = 0; +<<<<<<< HEAD mock_env_->set_current_time(0); // in seconds options.env = mock_env_.get(); int counter = 0; @@ -136,12 +170,38 @@ TEST_F(StatsHistoryTest, PersistentStatsFreshInstall) { ASSERT_EQ(5u, dbfull()->GetDBOptions().stats_persist_period_sec); dbfull()->TEST_WaitForStatsDumpRun([&] { mock_env_->set_current_time(5); }); +======= + std::unique_ptr mock_env; + mock_env.reset(new rocksdb::MockTimeEnv(env_)); + mock_env->set_current_time(0); // in seconds + options.env = mock_env.get(); + rocksdb::SyncPoint::GetInstance()->DisableProcessing(); + rocksdb::SyncPoint::GetInstance()->ClearAllCallBacks(); +#if defined(OS_MACOSX) && !defined(NDEBUG) + rocksdb::SyncPoint::GetInstance()->SetCallBack( + "InstrumentedCondVar::TimedWaitInternal", [&](void* arg) { + uint64_t time_us = *reinterpret_cast(arg); + if (time_us < mock_env->RealNowMicros()) { + *reinterpret_cast(arg) = mock_env->RealNowMicros() + 1000; + } + }); +#endif // OS_MACOSX && !NDEBUG + int counter = 0; + rocksdb::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::PersistStats:Entry", [&](void* /*arg*/) { counter++; }); + rocksdb::SyncPoint::GetInstance()->EnableProcessing(); + Reopen(options); + ASSERT_OK(dbfull()->SetDBOptions({{"stats_persist_period_sec", "5"}})); + ASSERT_EQ(5, dbfull()->GetDBOptions().stats_persist_period_sec); + dbfull()->TEST_WaitForPersistStatsRun([&] { mock_env->set_current_time(5); }); +>>>>>>> 671d15cbd... Persistent Stats: persist stats history to disk (#5046) ASSERT_GE(counter, 1); Close(); } // TODO(Zhongyi): Move persistent stats related tests to a separate file TEST_F(StatsHistoryTest, GetStatsHistoryInMemory) { +<<<<<<< HEAD constexpr int kPeriodSec = 5; Options options; options.create_if_missing = true; @@ -150,10 +210,34 @@ TEST_F(StatsHistoryTest, GetStatsHistoryInMemory) { int mock_time_sec = 0; mock_env_->set_current_time(mock_time_sec); options.env = mock_env_.get(); +======= + Options options; + options.create_if_missing = true; + options.stats_persist_period_sec = 5; + options.statistics = rocksdb::CreateDBStatistics(); + std::unique_ptr mock_env; + mock_env.reset(new rocksdb::MockTimeEnv(env_)); + mock_env->set_current_time(0); // in seconds + options.env = mock_env.get(); +#if defined(OS_MACOSX) && !defined(NDEBUG) + rocksdb::SyncPoint::GetInstance()->DisableProcessing(); + rocksdb::SyncPoint::GetInstance()->ClearAllCallBacks(); + rocksdb::SyncPoint::GetInstance()->SetCallBack( + "InstrumentedCondVar::TimedWaitInternal", [&](void* arg) { + uint64_t time_us = *reinterpret_cast(arg); + if (time_us < mock_env->RealNowMicros()) { + *reinterpret_cast(arg) = mock_env->RealNowMicros() + 1000; + } + }); + rocksdb::SyncPoint::GetInstance()->EnableProcessing(); +#endif // OS_MACOSX && !NDEBUG + +>>>>>>> 671d15cbd... Persistent Stats: persist stats history to disk (#5046) CreateColumnFamilies({"pikachu"}, options); ASSERT_OK(Put("foo", "bar")); ReopenWithColumnFamilies({"default", "pikachu"}, options); +<<<<<<< HEAD // make sure the first stats persist to finish mock_time_sec += kPeriodSec - 1; dbfull()->TEST_WaitForStatsDumpRun( @@ -166,22 +250,41 @@ TEST_F(StatsHistoryTest, GetStatsHistoryInMemory) { std::unique_ptr stats_iter; db_->GetStatsHistory(0, mock_time_sec + 1, &stats_iter); +======= + int mock_time = 1; + // Wait for stats persist to finish + dbfull()->TEST_WaitForPersistStatsRun([&] { mock_env->set_current_time(5); }); + std::unique_ptr stats_iter; + db_->GetStatsHistory(0 /*start_time*/, 6 /*end_time*/, &stats_iter); +>>>>>>> 671d15cbd... Persistent Stats: persist stats history to disk (#5046) ASSERT_TRUE(stats_iter != nullptr); // disabled stats snapshots ASSERT_OK(dbfull()->SetDBOptions({{"stats_persist_period_sec", "0"}})); size_t stats_count = 0; for (; stats_iter->Valid(); stats_iter->Next()) { auto stats_map = stats_iter->GetStatsMap(); +<<<<<<< HEAD ASSERT_EQ(stats_iter->GetStatsTime(), mock_time_sec); +======= + ASSERT_EQ(stats_iter->GetStatsTime(), 5); +>>>>>>> 671d15cbd... Persistent Stats: persist stats history to disk (#5046) stats_count += stats_map.size(); } ASSERT_GT(stats_count, 0); // Wait a bit and verify no more stats are found +<<<<<<< HEAD for (; mock_time_sec < 30; ++mock_time_sec) { dbfull()->TEST_WaitForStatsDumpRun( [&] { mock_env_->set_current_time(mock_time_sec); }); } db_->GetStatsHistory(0, mock_time_sec, &stats_iter); +======= + for (mock_time = 6; mock_time < 20; ++mock_time) { + dbfull()->TEST_WaitForPersistStatsRun( + [&] { mock_env->set_current_time(mock_time); }); + } + db_->GetStatsHistory(0 /*start_time*/, 20 /*end_time*/, &stats_iter); +>>>>>>> 671d15cbd... Persistent Stats: persist stats history to disk (#5046) ASSERT_TRUE(stats_iter != nullptr); size_t stats_count_new = 0; for (; stats_iter->Valid(); stats_iter->Next()) { @@ -192,6 +295,7 @@ TEST_F(StatsHistoryTest, GetStatsHistoryInMemory) { } TEST_F(StatsHistoryTest, InMemoryStatsHistoryPurging) { +<<<<<<< HEAD constexpr int kPeriodSec = 1; Options options; options.create_if_missing = true; @@ -200,6 +304,28 @@ TEST_F(StatsHistoryTest, InMemoryStatsHistoryPurging) { int mock_time_sec = 0; mock_env_->set_current_time(mock_time_sec); options.env = mock_env_.get(); +======= + Options options; + options.create_if_missing = true; + options.statistics = rocksdb::CreateDBStatistics(); + options.stats_persist_period_sec = 1; + std::unique_ptr mock_env; + mock_env.reset(new rocksdb::MockTimeEnv(env_)); + mock_env->set_current_time(0); // in seconds + options.env = mock_env.get(); +#if defined(OS_MACOSX) && !defined(NDEBUG) + rocksdb::SyncPoint::GetInstance()->DisableProcessing(); + rocksdb::SyncPoint::GetInstance()->ClearAllCallBacks(); + rocksdb::SyncPoint::GetInstance()->SetCallBack( + "InstrumentedCondVar::TimedWaitInternal", [&](void* arg) { + uint64_t time_us = *reinterpret_cast(arg); + if (time_us < mock_env->RealNowMicros()) { + *reinterpret_cast(arg) = mock_env->RealNowMicros() + 1000; + } + }); + rocksdb::SyncPoint::GetInstance()->EnableProcessing(); +#endif // OS_MACOSX && !NDEBUG +>>>>>>> 671d15cbd... Persistent Stats: persist stats history to disk (#5046) CreateColumnFamilies({"pikachu"}, options); ASSERT_OK(Put("foo", "bar")); @@ -220,10 +346,18 @@ TEST_F(StatsHistoryTest, InMemoryStatsHistoryPurging) { ASSERT_OK(Flush()); ASSERT_OK(Delete("sol")); db_->CompactRange(CompactRangeOptions(), nullptr, nullptr); +<<<<<<< HEAD // Wait for stats persist to finish for (mock_time_sec = 1; mock_time_sec < kPeriodSec; mock_time_sec++) { dbfull()->TEST_WaitForStatsDumpRun( [&] { mock_env_->set_current_time(mock_time_sec); }); +======= + int mock_time = 1; + // Wait for stats persist to finish + for (; mock_time < 5; ++mock_time) { + dbfull()->TEST_WaitForPersistStatsRun( + [&] { mock_env->set_current_time(mock_time); }); +>>>>>>> 671d15cbd... Persistent Stats: persist stats history to disk (#5046) } // second round of ops @@ -237,6 +371,7 @@ TEST_F(StatsHistoryTest, InMemoryStatsHistoryPurging) { delete iterator; ASSERT_OK(Flush()); db_->CompactRange(CompactRangeOptions(), nullptr, nullptr); +<<<<<<< HEAD for (; mock_time_sec < 10; mock_time_sec++) { dbfull()->TEST_WaitForStatsDumpRun( @@ -245,6 +380,14 @@ TEST_F(StatsHistoryTest, InMemoryStatsHistoryPurging) { std::unique_ptr stats_iter; db_->GetStatsHistory(0, 10, &stats_iter); +======= + for (; mock_time < 10; ++mock_time) { + dbfull()->TEST_WaitForPersistStatsRun( + [&] { mock_env->set_current_time(mock_time); }); + } + std::unique_ptr stats_iter; + db_->GetStatsHistory(0 /*start_time*/, 10 /*end_time*/, &stats_iter); +>>>>>>> 671d15cbd... Persistent Stats: persist stats history to disk (#5046) ASSERT_TRUE(stats_iter != nullptr); size_t stats_count = 0; int slice_count = 0; @@ -255,6 +398,7 @@ TEST_F(StatsHistoryTest, InMemoryStatsHistoryPurging) { } size_t stats_history_size = dbfull()->TEST_EstimateInMemoryStatsHistorySize(); ASSERT_GE(slice_count, 9); +<<<<<<< HEAD ASSERT_GE(stats_history_size, 13000); // capping memory cost at 13000 bytes since one slice is around 10000~13000 ASSERT_OK(dbfull()->SetDBOptions({{"stats_history_buffer_size", "13000"}})); @@ -267,6 +411,18 @@ TEST_F(StatsHistoryTest, InMemoryStatsHistoryPurging) { } db_->GetStatsHistory(0, 20, &stats_iter); +======= + ASSERT_GE(stats_history_size, 12000); + // capping memory cost at 12000 bytes since one slice is around 10000~12000 + ASSERT_OK(dbfull()->SetDBOptions({{"stats_history_buffer_size", "12000"}})); + ASSERT_EQ(12000, dbfull()->GetDBOptions().stats_history_buffer_size); + // Wait for stats persist to finish + for (; mock_time < 20; ++mock_time) { + dbfull()->TEST_WaitForPersistStatsRun( + [&] { mock_env->set_current_time(mock_time); }); + } + db_->GetStatsHistory(0 /*start_time*/, 20 /*end_time*/, &stats_iter); +>>>>>>> 671d15cbd... Persistent Stats: persist stats history to disk (#5046) ASSERT_TRUE(stats_iter != nullptr); size_t stats_count_reopen = 0; slice_count = 0; @@ -279,7 +435,11 @@ TEST_F(StatsHistoryTest, InMemoryStatsHistoryPurging) { dbfull()->TEST_EstimateInMemoryStatsHistorySize(); // only one slice can fit under the new stats_history_buffer_size ASSERT_LT(slice_count, 2); +<<<<<<< HEAD ASSERT_TRUE(stats_history_size_reopen < 13000 && +======= + ASSERT_TRUE(stats_history_size_reopen < 12000 && +>>>>>>> 671d15cbd... Persistent Stats: persist stats history to disk (#5046) stats_history_size_reopen > 0); ASSERT_TRUE(stats_count_reopen < stats_count && stats_count_reopen > 0); Close(); @@ -296,6 +456,7 @@ int countkeys(Iterator* iter) { } TEST_F(StatsHistoryTest, GetStatsHistoryFromDisk) { +<<<<<<< HEAD constexpr int kPeriodSec = 5; Options options; options.create_if_missing = true; @@ -305,11 +466,23 @@ TEST_F(StatsHistoryTest, GetStatsHistoryFromDisk) { int mock_time_sec = 0; mock_env_->set_current_time(mock_time_sec); options.env = mock_env_.get(); +======= + Options options; + options.create_if_missing = true; + options.stats_persist_period_sec = 5; + options.statistics = rocksdb::CreateDBStatistics(); + options.persist_stats_to_disk = true; + std::unique_ptr mock_env; + mock_env.reset(new rocksdb::MockTimeEnv(env_)); + mock_env->set_current_time(0); // in seconds + options.env = mock_env.get(); +>>>>>>> 671d15cbd... Persistent Stats: persist stats history to disk (#5046) CreateColumnFamilies({"pikachu"}, options); ASSERT_OK(Put("foo", "bar")); ReopenWithColumnFamilies({"default", "pikachu"}, options); ASSERT_EQ(Get("foo"), "bar"); +<<<<<<< HEAD // Wait for the first stats persist to finish, as the initial delay could be // different. mock_time_sec += kPeriodSec - 1; @@ -321,22 +494,36 @@ TEST_F(StatsHistoryTest, GetStatsHistoryFromDisk) { dbfull()->TEST_WaitForStatsDumpRun( [&] { mock_env_->set_current_time(mock_time_sec); }); +======= + // Wait for stats persist to finish + dbfull()->TEST_WaitForPersistStatsRun([&] { mock_env->set_current_time(5); }); +>>>>>>> 671d15cbd... Persistent Stats: persist stats history to disk (#5046) auto iter = db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily()); int key_count1 = countkeys(iter); delete iter; +<<<<<<< HEAD mock_time_sec += kPeriodSec; dbfull()->TEST_WaitForStatsDumpRun( [&] { mock_env_->set_current_time(mock_time_sec); }); +======= + dbfull()->TEST_WaitForPersistStatsRun( + [&] { mock_env->set_current_time(10); }); +>>>>>>> 671d15cbd... Persistent Stats: persist stats history to disk (#5046) iter = db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily()); int key_count2 = countkeys(iter); delete iter; +<<<<<<< HEAD mock_time_sec += kPeriodSec; dbfull()->TEST_WaitForStatsDumpRun( [&] { mock_env_->set_current_time(mock_time_sec); }); +======= + dbfull()->TEST_WaitForPersistStatsRun( + [&] { mock_env->set_current_time(15); }); +>>>>>>> 671d15cbd... Persistent Stats: persist stats history to disk (#5046) iter = db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily()); int key_count3 = countkeys(iter); @@ -345,15 +532,26 @@ TEST_F(StatsHistoryTest, GetStatsHistoryFromDisk) { ASSERT_GE(key_count3, key_count2); ASSERT_EQ(key_count3 - key_count2, key_count2 - key_count1); std::unique_ptr stats_iter; +<<<<<<< HEAD db_->GetStatsHistory(0, mock_time_sec + 1, &stats_iter); +======= + db_->GetStatsHistory(0 /*start_time*/, 16 /*end_time*/, &stats_iter); +>>>>>>> 671d15cbd... Persistent Stats: persist stats history to disk (#5046) ASSERT_TRUE(stats_iter != nullptr); size_t stats_count = 0; int slice_count = 0; int non_zero_count = 0; +<<<<<<< HEAD for (int i = 2; stats_iter->Valid(); stats_iter->Next(), i++) { slice_count++; auto stats_map = stats_iter->GetStatsMap(); ASSERT_EQ(stats_iter->GetStatsTime(), kPeriodSec * i - 1); +======= + for (int i = 1; stats_iter->Valid(); stats_iter->Next(), i++) { + slice_count++; + auto stats_map = stats_iter->GetStatsMap(); + ASSERT_EQ(stats_iter->GetStatsTime(), 5 * i); +>>>>>>> 671d15cbd... Persistent Stats: persist stats history to disk (#5046) for (auto& stat : stats_map) { if (stat.second != 0) { non_zero_count++; @@ -366,7 +564,11 @@ TEST_F(StatsHistoryTest, GetStatsHistoryFromDisk) { ASSERT_EQ(stats_count, key_count3 - 2); // verify reopen will not cause data loss ReopenWithColumnFamilies({"default", "pikachu"}, options); +<<<<<<< HEAD db_->GetStatsHistory(0, mock_time_sec + 1, &stats_iter); +======= + db_->GetStatsHistory(0 /*start_time*/, 16 /*end_time*/, &stats_iter); +>>>>>>> 671d15cbd... Persistent Stats: persist stats history to disk (#5046) ASSERT_TRUE(stats_iter != nullptr); size_t stats_count_reopen = 0; int slice_count_reopen = 0; @@ -381,7 +583,10 @@ TEST_F(StatsHistoryTest, GetStatsHistoryFromDisk) { } stats_count_reopen += stats_map.size(); } +<<<<<<< HEAD +======= +>>>>>>> 671d15cbd... Persistent Stats: persist stats history to disk (#5046) ASSERT_EQ(non_zero_count, non_zero_count_recover); ASSERT_EQ(slice_count, slice_count_reopen); ASSERT_EQ(stats_count, stats_count_reopen); @@ -391,6 +596,7 @@ TEST_F(StatsHistoryTest, GetStatsHistoryFromDisk) { // Test persisted stats matches the value found in options.statistics and // the stats value retains after DB reopen TEST_F(StatsHistoryTest, PersitentStatsVerifyValue) { +<<<<<<< HEAD constexpr int kPeriodSec = 5; Options options; options.create_if_missing = true; @@ -403,11 +609,25 @@ TEST_F(StatsHistoryTest, PersitentStatsVerifyValue) { ASSERT_TRUE(options.statistics->getTickerMap(&stats_map_before)); mock_env_->set_current_time(mock_time_sec); options.env = mock_env_.get(); +======= + Options options; + options.create_if_missing = true; + options.stats_persist_period_sec = 5; + options.statistics = rocksdb::CreateDBStatistics(); + options.persist_stats_to_disk = true; + std::unique_ptr mock_env; + mock_env.reset(new rocksdb::MockTimeEnv(env_)); + std::map stats_map_before; + ASSERT_TRUE(options.statistics->getTickerMap(&stats_map_before)); + mock_env->set_current_time(0); // in seconds + options.env = mock_env.get(); +>>>>>>> 671d15cbd... Persistent Stats: persist stats history to disk (#5046) CreateColumnFamilies({"pikachu"}, options); ASSERT_OK(Put("foo", "bar")); ReopenWithColumnFamilies({"default", "pikachu"}, options); ASSERT_EQ(Get("foo"), "bar"); +<<<<<<< HEAD // Wait for the first stats persist to finish, as the initial delay could be // different. mock_time_sec += kPeriodSec - 1; @@ -418,34 +638,54 @@ TEST_F(StatsHistoryTest, PersitentStatsVerifyValue) { mock_time_sec += kPeriodSec; dbfull()->TEST_WaitForStatsDumpRun( [&] { mock_env_->set_current_time(mock_time_sec); }); +======= + // Wait for stats persist to finish + dbfull()->TEST_WaitForPersistStatsRun([&] { mock_env->set_current_time(5); }); +>>>>>>> 671d15cbd... Persistent Stats: persist stats history to disk (#5046) auto iter = db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily()); countkeys(iter); delete iter; +<<<<<<< HEAD mock_time_sec += kPeriodSec; dbfull()->TEST_WaitForStatsDumpRun( [&] { mock_env_->set_current_time(mock_time_sec); }); +======= + dbfull()->TEST_WaitForPersistStatsRun( + [&] { mock_env->set_current_time(10); }); +>>>>>>> 671d15cbd... Persistent Stats: persist stats history to disk (#5046) iter = db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily()); countkeys(iter); delete iter; +<<<<<<< HEAD mock_time_sec += kPeriodSec; dbfull()->TEST_WaitForStatsDumpRun( [&] { mock_env_->set_current_time(mock_time_sec); }); +======= + dbfull()->TEST_WaitForPersistStatsRun( + [&] { mock_env->set_current_time(15); }); +>>>>>>> 671d15cbd... Persistent Stats: persist stats history to disk (#5046) iter = db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily()); countkeys(iter); delete iter; +<<<<<<< HEAD mock_time_sec += kPeriodSec; dbfull()->TEST_WaitForStatsDumpRun( [&] { mock_env_->set_current_time(mock_time_sec); }); +======= + dbfull()->TEST_WaitForPersistStatsRun( + [&] { mock_env->set_current_time(20); }); +>>>>>>> 671d15cbd... Persistent Stats: persist stats history to disk (#5046) std::map stats_map_after; ASSERT_TRUE(options.statistics->getTickerMap(&stats_map_after)); std::unique_ptr stats_iter; +<<<<<<< HEAD db_->GetStatsHistory(0, mock_time_sec + 1, &stats_iter); ASSERT_TRUE(stats_iter != nullptr); std::string sample = "rocksdb.num.iterator.deleted"; @@ -453,6 +693,15 @@ TEST_F(StatsHistoryTest, PersitentStatsVerifyValue) { for (int i = 2; stats_iter->Valid(); stats_iter->Next(), ++i) { auto stats_map = stats_iter->GetStatsMap(); ASSERT_EQ(stats_iter->GetStatsTime(), kPeriodSec * i - 1); +======= + db_->GetStatsHistory(0 /*start_time*/, 21 /*end_time*/, &stats_iter); + ASSERT_TRUE(stats_iter != nullptr); + std::string sample = "rocksdb.num.iterator.deleted"; + uint64_t recovered_value = 0; + for (int i = 1; stats_iter->Valid(); stats_iter->Next(), ++i) { + auto stats_map = stats_iter->GetStatsMap(); + ASSERT_EQ(stats_iter->GetStatsTime(), 5 * i); +>>>>>>> 671d15cbd... Persistent Stats: persist stats history to disk (#5046) for (const auto& stat : stats_map) { if (sample.compare(stat.first) == 0) { recovered_value += stat.second; @@ -463,12 +712,21 @@ TEST_F(StatsHistoryTest, PersitentStatsVerifyValue) { // test stats value retains after recovery ReopenWithColumnFamilies({"default", "pikachu"}, options); +<<<<<<< HEAD db_->GetStatsHistory(0, mock_time_sec + 1, &stats_iter); ASSERT_TRUE(stats_iter != nullptr); uint64_t new_recovered_value = 0; for (int i = 2; stats_iter->Valid(); stats_iter->Next(), i++) { auto stats_map = stats_iter->GetStatsMap(); ASSERT_EQ(stats_iter->GetStatsTime(), kPeriodSec * i - 1); +======= + db_->GetStatsHistory(0 /*start_time*/, 21 /*end_time*/, &stats_iter); + ASSERT_TRUE(stats_iter != nullptr); + uint64_t new_recovered_value = 0; + for (int i = 1; stats_iter->Valid(); stats_iter->Next(), i++) { + auto stats_map = stats_iter->GetStatsMap(); + ASSERT_EQ(stats_iter->GetStatsTime(), 5 * i); +>>>>>>> 671d15cbd... Persistent Stats: persist stats history to disk (#5046) for (const auto& stat : stats_map) { if (sample.compare(stat.first) == 0) { new_recovered_value += stat.second; @@ -485,6 +743,7 @@ TEST_F(StatsHistoryTest, PersitentStatsVerifyValue) { // TODO(Zhongyi): add test for different format versions TEST_F(StatsHistoryTest, PersistentStatsCreateColumnFamilies) { +<<<<<<< HEAD constexpr int kPeriodSec = 5; Options options; options.create_if_missing = true; @@ -494,6 +753,17 @@ TEST_F(StatsHistoryTest, PersistentStatsCreateColumnFamilies) { int mock_time_sec = 0; mock_env_->set_current_time(mock_time_sec); options.env = mock_env_.get(); +======= + Options options; + options.create_if_missing = true; + options.stats_persist_period_sec = 5; + options.statistics = rocksdb::CreateDBStatistics(); + options.persist_stats_to_disk = true; + std::unique_ptr mock_env; + mock_env.reset(new rocksdb::MockTimeEnv(env_)); + mock_env->set_current_time(0); // in seconds + options.env = mock_env.get(); +>>>>>>> 671d15cbd... Persistent Stats: persist stats history to disk (#5046) ASSERT_OK(TryReopen(options)); CreateColumnFamilies({"one", "two", "three"}, options); ASSERT_OK(Put(1, "foo", "bar")); @@ -502,6 +772,7 @@ TEST_F(StatsHistoryTest, PersistentStatsCreateColumnFamilies) { CreateColumnFamilies({"four"}, options); ReopenWithColumnFamilies({"default", "one", "two", "three", "four"}, options); ASSERT_EQ(Get(2, "foo"), "bar"); +<<<<<<< HEAD // make sure the first stats persist to finish mock_time_sec += kPeriodSec - 1; @@ -511,6 +782,9 @@ TEST_F(StatsHistoryTest, PersistentStatsCreateColumnFamilies) { mock_time_sec += kPeriodSec; dbfull()->TEST_WaitForStatsDumpRun( [&] { mock_env_->set_current_time(mock_time_sec); }); +======= + dbfull()->TEST_WaitForPersistStatsRun([&] { mock_env->set_current_time(5); }); +>>>>>>> 671d15cbd... Persistent Stats: persist stats history to disk (#5046) auto iter = db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily()); int key_count = countkeys(iter); @@ -519,7 +793,11 @@ TEST_F(StatsHistoryTest, PersistentStatsCreateColumnFamilies) { uint64_t num_write_wal = 0; std::string sample = "rocksdb.write.wal"; std::unique_ptr stats_iter; +<<<<<<< HEAD db_->GetStatsHistory(0, mock_time_sec, &stats_iter); +======= + db_->GetStatsHistory(0 /*start_time*/, 5 /*end_time*/, &stats_iter); +>>>>>>> 671d15cbd... Persistent Stats: persist stats history to disk (#5046) ASSERT_TRUE(stats_iter != nullptr); for (; stats_iter->Valid(); stats_iter->Next()) { auto stats_map = stats_iter->GetStatsMap(); @@ -555,7 +833,11 @@ TEST_F(StatsHistoryTest, PersistentStatsCreateColumnFamilies) { ASSERT_NOK(db_->CreateColumnFamily(cf_opts, kPersistentStatsColumnFamilyName, &handle)); // verify stats is not affected by prior failed CF creation +<<<<<<< HEAD db_->GetStatsHistory(0, mock_time_sec, &stats_iter); +======= + db_->GetStatsHistory(0 /*start_time*/, 5 /*end_time*/, &stats_iter); +>>>>>>> 671d15cbd... Persistent Stats: persist stats history to disk (#5046) ASSERT_TRUE(stats_iter != nullptr); num_write_wal = 0; for (; stats_iter->Valid(); stats_iter->Next()) { @@ -585,13 +867,18 @@ TEST_F(StatsHistoryTest, PersistentStatsReadOnly) { Close(); // Reopen and flush memtable. +<<<<<<< HEAD ASSERT_OK(TryReopen(options)); +======= + Reopen(options); +>>>>>>> 671d15cbd... Persistent Stats: persist stats history to disk (#5046) Flush(); Close(); // Now check keys in read only mode. ASSERT_OK(ReadOnlyReopen(options)); } +<<<<<<< HEAD TEST_F(StatsHistoryTest, ForceManualFlushStatsCF) { constexpr int kPeriodSec = 5; Options options; @@ -686,6 +973,12 @@ TEST_F(StatsHistoryTest, ForceManualFlushStatsCF) { int main(int argc, char** argv) { ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); +======= +} // namespace rocksdb + +int main(int argc, char** argv) { + rocksdb::port::InstallStackTraceHandler(); +>>>>>>> 671d15cbd... Persistent Stats: persist stats history to disk (#5046) ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } diff --git a/options/db_options.cc b/options/db_options.cc index e145185ac..4d2f7df3f 100644 --- a/options/db_options.cc +++ b/options/db_options.cc @@ -93,7 +93,8 @@ ImmutableDBOptions::ImmutableDBOptions(const DBOptions& options) two_write_queues(options.two_write_queues), manual_wal_flush(options.manual_wal_flush), atomic_flush(options.atomic_flush), - avoid_unnecessary_blocking_io(options.avoid_unnecessary_blocking_io) { + avoid_unnecessary_blocking_io(options.avoid_unnecessary_blocking_io), + persist_stats_to_disk(options.persist_stats_to_disk) { } void ImmutableDBOptions::Dump(Logger* log) const { @@ -247,6 +248,8 @@ void ImmutableDBOptions::Dump(Logger* log) const { atomic_flush); ROCKS_LOG_HEADER(log, " Options.avoid_unnecessary_blocking_io: %d", avoid_unnecessary_blocking_io); + ROCKS_LOG_HEADER(log, " Options.persist_stats_to_disk: %u", + persist_stats_to_disk); } MutableDBOptions::MutableDBOptions() diff --git a/options/db_options.h b/options/db_options.h index 8042fb5bc..4bde22d21 100644 --- a/options/db_options.h +++ b/options/db_options.h @@ -87,6 +87,7 @@ struct ImmutableDBOptions { bool manual_wal_flush; bool atomic_flush; bool avoid_unnecessary_blocking_io; + bool persist_stats_to_disk; }; struct MutableDBOptions { diff --git a/options/options_helper.cc b/options/options_helper.cc index dfbdfb401..e756c25c1 100644 --- a/options/options_helper.cc +++ b/options/options_helper.cc @@ -90,6 +90,7 @@ DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options, options.stats_dump_period_sec = mutable_db_options.stats_dump_period_sec; options.stats_persist_period_sec = mutable_db_options.stats_persist_period_sec; + options.persist_stats_to_disk = immutable_db_options.persist_stats_to_disk; options.stats_history_buffer_size = mutable_db_options.stats_history_buffer_size; options.advise_random_on_open = immutable_db_options.advise_random_on_open; @@ -1626,6 +1627,10 @@ std::unordered_map {offsetof(struct DBOptions, stats_persist_period_sec), OptionType::kUInt, OptionVerificationType::kNormal, true, offsetof(struct MutableDBOptions, stats_persist_period_sec)}}, + {"persist_stats_to_disk", + {offsetof(struct DBOptions, persist_stats_to_disk), + OptionType::kBoolean, OptionVerificationType::kNormal, false, + offsetof(struct ImmutableDBOptions, persist_stats_to_disk)}}, {"stats_history_buffer_size", {offsetof(struct DBOptions, stats_history_buffer_size), OptionType::kSizeT, OptionVerificationType::kNormal, true, diff --git a/options/options_settable_test.cc b/options/options_settable_test.cc index fa1e6709a..eaf3b1f4f 100644 --- a/options/options_settable_test.cc +++ b/options/options_settable_test.cc @@ -275,6 +275,7 @@ TEST_F(OptionsSettableTest, DBOptionsAllFieldsSettable) { "allow_mmap_writes=false;" "stats_dump_period_sec=70127;" "stats_persist_period_sec=54321;" + "persist_stats_to_disk=true;" "stats_history_buffer_size=14159;" "allow_fallocate=true;" "allow_mmap_reads=false;" diff --git a/options/options_test.cc b/options/options_test.cc index b751c8d6b..0d6d72b7f 100644 --- a/options/options_test.cc +++ b/options/options_test.cc @@ -135,6 +135,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) { {"skip_log_error_on_recovery", "false"}, {"stats_dump_period_sec", "46"}, {"stats_persist_period_sec", "57"}, + {"persist_stats_to_disk", "false"}, {"stats_history_buffer_size", "69"}, {"advise_random_on_open", "true"}, {"allow_mmap_populate", "true"}, @@ -275,6 +276,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) { ASSERT_EQ(new_db_opt.skip_log_error_on_recovery, false); ASSERT_EQ(new_db_opt.stats_dump_period_sec, 46U); ASSERT_EQ(new_db_opt.stats_persist_period_sec, 57U); + ASSERT_EQ(new_db_opt.persist_stats_to_disk, false); ASSERT_EQ(new_db_opt.stats_history_buffer_size, 69U); ASSERT_EQ(new_db_opt.advise_random_on_open, true); ASSERT_EQ(new_db_opt.allow_mmap_populate, true); diff --git a/src.mk b/src.mk index 2c543e3dc..83a2da112 100644 --- a/src.mk +++ b/src.mk @@ -36,7 +36,6 @@ LIB_SOURCES = \ db/flush_job.cc \ db/flush_scheduler.cc \ db/forward_iterator.cc \ - db/in_memory_stats_history.cc \ db/internal_stats.cc \ db/logs_with_prep_tracker.cc \ db/log_reader.cc \ @@ -82,10 +81,12 @@ LIB_SOURCES = \ memtable/write_buffer_manager.cc \ monitoring/histogram.cc \ monitoring/histogram_windowing.cc \ + monitoring/in_memory_stats_history.cc \ monitoring/instrumented_mutex.cc \ monitoring/iostats_context.cc \ monitoring/perf_context.cc \ monitoring/perf_level.cc \ + monitoring/persistent_stats_history.cc \ monitoring/statistics.cc \ monitoring/stats_dump_scheduler.cc \ monitoring/thread_status_impl.cc \ @@ -101,6 +102,7 @@ LIB_SOURCES = \ options/options_sanity_check.cc \ port/port_posix.cc \ port/stack_trace.cc \ +<<<<<<< HEAD table/adaptive_table_factory.cc \ table/block.cc \ table/block_based_filter_block.cc \ @@ -117,6 +119,27 @@ LIB_SOURCES = \ table/data_block_hash_index.cc \ table/data_block_footer.cc \ table/flush_block_policy.cc \ +======= + table/adaptive/adaptive_table_factory.cc \ + table/block_based/block.cc \ + table/block_based/block_based_filter_block.cc \ + table/block_based/block_based_table_builder.cc \ + table/block_based/block_based_table_factory.cc \ + table/block_based/block_based_table_reader.cc \ + table/block_based/block_builder.cc \ + table/block_based/block_prefix_index.cc \ + table/block_based/data_block_hash_index.cc \ + table/block_based/data_block_footer.cc \ + table/block_based/flush_block_policy.cc \ + table/block_based/full_filter_block.cc \ + table/block_based/index_builder.cc \ + table/block_based/partitioned_filter_block.cc \ + table/block_fetcher.cc \ + table/bloom_block.cc \ + table/cuckoo/cuckoo_table_builder.cc \ + table/cuckoo/cuckoo_table_factory.cc \ + table/cuckoo/cuckoo_table_reader.cc \ +>>>>>>> 671d15cbd... Persistent Stats: persist stats history to disk (#5046) table/format.cc \ table/full_filter_block.cc \ table/get_context.cc \ @@ -249,21 +272,32 @@ LIB_SOURCES_ASM = LIB_SOURCES_C = endif -TOOL_LIB_SOURCES = \ +TOOL_LIB_SOURCES = \ tools/ldb_cmd.cc \ tools/ldb_tool.cc \ tools/sst_dump_tool.cc \ +<<<<<<< HEAD ANALYZER_LIB_SOURCES = \ tools/trace_analyzer_tool.cc \ MOCK_LIB_SOURCES = \ table/mock_table.cc \ util/fault_injection_test_env.cc +======= +ANALYZER_LIB_SOURCES = \ + tools/block_cache_trace_analyzer.cc \ + tools/trace_analyzer_tool.cc \ + +MOCK_LIB_SOURCES = \ + table/mock_table.cc \ + test_util/fault_injection_test_env.cc +>>>>>>> 671d15cbd... Persistent Stats: persist stats history to disk (#5046) -BENCH_LIB_SOURCES = \ +BENCH_LIB_SOURCES = \ tools/db_bench_tool.cc \ +<<<<<<< HEAD EXP_LIB_SOURCES = \ utilities/col_buf_decoder.cc \ utilities/col_buf_encoder.cc \ @@ -273,6 +307,12 @@ TEST_LIB_SOURCES = \ db/db_test_util.cc \ util/testharness.cc \ util/testutil.cc \ +======= +TEST_LIB_SOURCES = \ + db/db_test_util.cc \ + test_util/testharness.cc \ + test_util/testutil.cc \ +>>>>>>> 671d15cbd... Persistent Stats: persist stats history to disk (#5046) utilities/cassandra/test_utils.cc \ MAIN_SOURCES = \ @@ -370,12 +410,22 @@ MAIN_SOURCES = \ monitoring/histogram_test.cc \ monitoring/iostats_context_test.cc \ monitoring/statistics_test.cc \ +<<<<<<< HEAD monitoring/stats_dump_scheduler_test.cc \ monitoring/stats_history_test.cc \ options/options_settable_test.cc options/options_test.cc \ table/block_based_filter_block_test.cc \ table/block_test.cc \ +======= + monitoring/stats_history_test.cc \ + options/options_test.cc \ + table/block_based/block_based_filter_block_test.cc \ + table/block_based/block_test.cc \ + table/block_based/data_block_hash_index_test.cc \ + table/block_based/full_filter_block_test.cc \ + table/block_based/partitioned_filter_block_test.cc \ +>>>>>>> 671d15cbd... Persistent Stats: persist stats history to disk (#5046) table/cleanable_test.cc \ table/cuckoo_table_builder_test.cc \ table/cuckoo_table_reader_test.cc \ @@ -396,9 +446,14 @@ MAIN_SOURCES = \ tools/ldb_cmd_test.cc \ tools/reduce_levels_test.cc \ tools/sst_dump_test.cc \ +<<<<<<< HEAD tools/trace_analyzer_test.cc \ util/arena_test.cc \ util/auto_roll_logger_test.cc \ +======= + tools/trace_analyzer_test.cc \ + trace_replay/block_cache_tracer_test.cc \ +>>>>>>> 671d15cbd... Persistent Stats: persist stats history to disk (#5046) util/autovector_test.cc \ util/bloom_test.cc \ util/coding_test.cc \ diff --git a/tools/db_bench_tool.cc b/tools/db_bench_tool.cc index 70ad188cc..4deb6d04b 100644 --- a/tools/db_bench_tool.cc +++ b/tools/db_bench_tool.cc @@ -1064,6 +1064,8 @@ DEFINE_uint64(stats_dump_period_sec, TERARKDB_NAMESPACE::Options().stats_dump_pe DEFINE_uint64(stats_persist_period_sec, rocksdb::Options().stats_persist_period_sec, "Gap between persisting stats in seconds"); +DEFINE_bool(persist_stats_to_disk, rocksdb::Options().persist_stats_to_disk, + "whether to persist stats to disk"); DEFINE_uint64(stats_history_buffer_size, rocksdb::Options().stats_history_buffer_size, "Max number of stats snapshots to keep in memory"); @@ -3569,6 +3571,7 @@ class Benchmark { static_cast(FLAGS_stats_dump_period_sec); options.stats_persist_period_sec = static_cast(FLAGS_stats_persist_period_sec); + options.persist_stats_to_disk = FLAGS_persist_stats_to_disk; options.stats_history_buffer_size = static_cast(FLAGS_stats_history_buffer_size); -- GitLab