From 1e73b11af7530acd1606925ac47a737be20379cf Mon Sep 17 00:00:00 2001 From: Yueh-Hsuan Chiang Date: Tue, 29 Sep 2015 17:13:02 -0700 Subject: [PATCH] Better handling of deprecated options in RocksDBOptionsParser Summary: Previously, we treat deprecated options as normal options in RocksDBOptionsParser. However, these deprecated options should not be verified and serialized. Test Plan: options_test Reviewers: igor, sdong, IslamAbdelRahman, anthony Reviewed By: anthony Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D47775 --- util/options_helper.cc | 3 + util/options_helper.h | 208 +++++++++++++++++++++++++---------------- util/options_parser.cc | 10 ++ 3 files changed, 138 insertions(+), 83 deletions(-) diff --git a/util/options_helper.cc b/util/options_helper.cc index 3e88095ce..d745d1c99 100644 --- a/util/options_helper.cc +++ b/util/options_helper.cc @@ -702,6 +702,9 @@ Status GetStringFromColumnFamilyOptions(std::string* opt_string, opt_string->clear(); for (auto iter = cf_options_type_info.begin(); iter != cf_options_type_info.end(); ++iter) { + if (iter->second.verification == OptionVerificationType::kDeprecated) { + continue; + } std::string single_output; bool result = SerializeSingleColumnFamilyOption(&single_output, cf_options, iter->first, delimiter); diff --git a/util/options_helper.h b/util/options_helper.h index 9190f8edc..38613ec45 100644 --- a/util/options_helper.h +++ b/util/options_helper.h @@ -68,11 +68,21 @@ enum class OptionType { kUnknown }; +enum class OptionVerificationType { + kNormal, + kDeprecated // The option is no longer used in rocksdb. The RocksDB + // OptionsParser will still accept this option if it + // happen to exists in some Options file. However, the + // parser will not include it in serialization and + // verification processes. +}; + // A struct for storing constant option information such as option name, // option type, and offset. struct OptionTypeInfo { int offset; OptionType type; + OptionVerificationType verification; }; static std::unordered_map db_options_type_info = { @@ -91,94 +101,125 @@ static std::unordered_map db_options_type_info = { std::vector> listeners; */ {"advise_random_on_open", - {offsetof(struct DBOptions, advise_random_on_open), OptionType::kBoolean}}, + {offsetof(struct DBOptions, advise_random_on_open), OptionType::kBoolean, + OptionVerificationType::kNormal}}, {"allow_mmap_reads", - {offsetof(struct DBOptions, allow_mmap_reads), OptionType::kBoolean}}, + {offsetof(struct DBOptions, allow_mmap_reads), OptionType::kBoolean, + OptionVerificationType::kNormal}}, {"allow_mmap_writes", - {offsetof(struct DBOptions, allow_mmap_writes), OptionType::kBoolean}}, + {offsetof(struct DBOptions, allow_mmap_writes), OptionType::kBoolean, + OptionVerificationType::kNormal}}, {"allow_os_buffer", - {offsetof(struct DBOptions, allow_os_buffer), OptionType::kBoolean}}, + {offsetof(struct DBOptions, allow_os_buffer), OptionType::kBoolean, + OptionVerificationType::kNormal}}, {"create_if_missing", - {offsetof(struct DBOptions, create_if_missing), OptionType::kBoolean}}, + {offsetof(struct DBOptions, create_if_missing), OptionType::kBoolean, + OptionVerificationType::kNormal}}, {"create_missing_column_families", {offsetof(struct DBOptions, create_missing_column_families), - OptionType::kBoolean}}, + OptionType::kBoolean, OptionVerificationType::kNormal}}, {"disableDataSync", - {offsetof(struct DBOptions, disableDataSync), OptionType::kBoolean}}, + {offsetof(struct DBOptions, disableDataSync), OptionType::kBoolean, + OptionVerificationType::kNormal}}, {"disable_data_sync", // for compatibility - {offsetof(struct DBOptions, disableDataSync), OptionType::kBoolean}}, + {offsetof(struct DBOptions, disableDataSync), OptionType::kBoolean, + OptionVerificationType::kNormal}}, {"enable_thread_tracking", - {offsetof(struct DBOptions, enable_thread_tracking), - OptionType::kBoolean}}, + {offsetof(struct DBOptions, enable_thread_tracking), OptionType::kBoolean, + OptionVerificationType::kNormal}}, {"error_if_exists", - {offsetof(struct DBOptions, error_if_exists), OptionType::kBoolean}}, + {offsetof(struct DBOptions, error_if_exists), OptionType::kBoolean, + OptionVerificationType::kNormal}}, {"is_fd_close_on_exec", - {offsetof(struct DBOptions, is_fd_close_on_exec), OptionType::kBoolean}}, + {offsetof(struct DBOptions, is_fd_close_on_exec), OptionType::kBoolean, + OptionVerificationType::kNormal}}, {"paranoid_checks", - {offsetof(struct DBOptions, paranoid_checks), OptionType::kBoolean}}, + {offsetof(struct DBOptions, paranoid_checks), OptionType::kBoolean, + OptionVerificationType::kNormal}}, {"skip_log_error_on_recovery", {offsetof(struct DBOptions, skip_log_error_on_recovery), - OptionType::kBoolean}}, + OptionType::kBoolean, OptionVerificationType::kNormal}}, {"skip_stats_update_on_db_open", {offsetof(struct DBOptions, skip_stats_update_on_db_open), - OptionType::kBoolean}}, + OptionType::kBoolean, OptionVerificationType::kNormal}}, {"new_table_reader_for_compaction_inputs", {offsetof(struct DBOptions, new_table_reader_for_compaction_inputs), - OptionType::kBoolean}}, + OptionType::kBoolean, OptionVerificationType::kNormal}}, {"compaction_readahead_size", - {offsetof(struct DBOptions, compaction_readahead_size), - OptionType::kSizeT}}, + {offsetof(struct DBOptions, compaction_readahead_size), OptionType::kSizeT, + OptionVerificationType::kNormal}}, {"use_adaptive_mutex", - {offsetof(struct DBOptions, use_adaptive_mutex), OptionType::kBoolean}}, + {offsetof(struct DBOptions, use_adaptive_mutex), OptionType::kBoolean, + OptionVerificationType::kNormal}}, {"use_fsync", - {offsetof(struct DBOptions, use_fsync), OptionType::kBoolean}}, + {offsetof(struct DBOptions, use_fsync), OptionType::kBoolean, + OptionVerificationType::kNormal}}, {"max_background_compactions", - {offsetof(struct DBOptions, max_background_compactions), - OptionType::kInt}}, + {offsetof(struct DBOptions, max_background_compactions), OptionType::kInt, + OptionVerificationType::kNormal}}, {"max_background_flushes", - {offsetof(struct DBOptions, max_background_flushes), OptionType::kInt}}, + {offsetof(struct DBOptions, max_background_flushes), OptionType::kInt, + OptionVerificationType::kNormal}}, {"max_file_opening_threads", - {offsetof(struct DBOptions, max_file_opening_threads), OptionType::kInt}}, + {offsetof(struct DBOptions, max_file_opening_threads), OptionType::kInt, + OptionVerificationType::kNormal}}, {"max_open_files", - {offsetof(struct DBOptions, max_open_files), OptionType::kInt}}, + {offsetof(struct DBOptions, max_open_files), OptionType::kInt, + OptionVerificationType::kNormal}}, {"table_cache_numshardbits", - {offsetof(struct DBOptions, table_cache_numshardbits), OptionType::kInt}}, + {offsetof(struct DBOptions, table_cache_numshardbits), OptionType::kInt, + OptionVerificationType::kNormal}}, {"db_write_buffer_size", - {offsetof(struct DBOptions, db_write_buffer_size), OptionType::kSizeT}}, + {offsetof(struct DBOptions, db_write_buffer_size), OptionType::kSizeT, + OptionVerificationType::kNormal}}, {"keep_log_file_num", - {offsetof(struct DBOptions, keep_log_file_num), OptionType::kSizeT}}, + {offsetof(struct DBOptions, keep_log_file_num), OptionType::kSizeT, + OptionVerificationType::kNormal}}, {"log_file_time_to_roll", - {offsetof(struct DBOptions, log_file_time_to_roll), OptionType::kSizeT}}, + {offsetof(struct DBOptions, log_file_time_to_roll), OptionType::kSizeT, + OptionVerificationType::kNormal}}, {"manifest_preallocation_size", {offsetof(struct DBOptions, manifest_preallocation_size), - OptionType::kSizeT}}, + OptionType::kSizeT, OptionVerificationType::kNormal}}, {"max_log_file_size", - {offsetof(struct DBOptions, max_log_file_size), OptionType::kSizeT}}, + {offsetof(struct DBOptions, max_log_file_size), OptionType::kSizeT, + OptionVerificationType::kNormal}}, {"db_log_dir", - {offsetof(struct DBOptions, db_log_dir), OptionType::kString}}, - {"wal_dir", {offsetof(struct DBOptions, wal_dir), OptionType::kString}}, + {offsetof(struct DBOptions, db_log_dir), OptionType::kString, + OptionVerificationType::kNormal}}, + {"wal_dir", + {offsetof(struct DBOptions, wal_dir), OptionType::kString, + OptionVerificationType::kNormal}}, {"max_subcompactions", - {offsetof(struct DBOptions, max_subcompactions), OptionType::kUInt32T}}, + {offsetof(struct DBOptions, max_subcompactions), OptionType::kUInt32T, + OptionVerificationType::kNormal}}, {"WAL_size_limit_MB", - {offsetof(struct DBOptions, WAL_size_limit_MB), OptionType::kUInt64T}}, + {offsetof(struct DBOptions, WAL_size_limit_MB), OptionType::kUInt64T, + OptionVerificationType::kNormal}}, {"WAL_ttl_seconds", - {offsetof(struct DBOptions, WAL_ttl_seconds), OptionType::kUInt64T}}, + {offsetof(struct DBOptions, WAL_ttl_seconds), OptionType::kUInt64T, + OptionVerificationType::kNormal}}, {"bytes_per_sync", - {offsetof(struct DBOptions, bytes_per_sync), OptionType::kUInt64T}}, + {offsetof(struct DBOptions, bytes_per_sync), OptionType::kUInt64T, + OptionVerificationType::kNormal}}, {"delayed_write_rate", - {offsetof(struct DBOptions, delayed_write_rate), OptionType::kUInt64T}}, + {offsetof(struct DBOptions, delayed_write_rate), OptionType::kUInt64T, + OptionVerificationType::kNormal}}, {"delete_obsolete_files_period_micros", {offsetof(struct DBOptions, delete_obsolete_files_period_micros), - OptionType::kUInt64T}}, + OptionType::kUInt64T, OptionVerificationType::kNormal}}, {"max_manifest_file_size", - {offsetof(struct DBOptions, max_manifest_file_size), - OptionType::kUInt64T}}, + {offsetof(struct DBOptions, max_manifest_file_size), OptionType::kUInt64T, + OptionVerificationType::kNormal}}, {"max_total_wal_size", - {offsetof(struct DBOptions, max_total_wal_size), OptionType::kUInt64T}}, + {offsetof(struct DBOptions, max_total_wal_size), OptionType::kUInt64T, + OptionVerificationType::kNormal}}, {"wal_bytes_per_sync", - {offsetof(struct DBOptions, wal_bytes_per_sync), OptionType::kUInt64T}}, + {offsetof(struct DBOptions, wal_bytes_per_sync), OptionType::kUInt64T, + OptionVerificationType::kNormal}}, {"stats_dump_period_sec", - {offsetof(struct DBOptions, stats_dump_period_sec), OptionType::kUInt}}}; + {offsetof(struct DBOptions, stats_dump_period_sec), OptionType::kUInt, + OptionVerificationType::kNormal}}}; static std::unordered_map cf_options_type_info = { /* not yet supported @@ -205,122 +246,123 @@ static std::unordered_map cf_options_type_info = { */ {"compaction_measure_io_stats", {offsetof(struct ColumnFamilyOptions, compaction_measure_io_stats), - OptionType::kBoolean}}, + OptionType::kBoolean, OptionVerificationType::kNormal}}, {"disable_auto_compactions", {offsetof(struct ColumnFamilyOptions, disable_auto_compactions), - OptionType::kBoolean}}, + OptionType::kBoolean, OptionVerificationType::kNormal}}, {"filter_deletes", {offsetof(struct ColumnFamilyOptions, filter_deletes), - OptionType::kBoolean}}, + OptionType::kBoolean, OptionVerificationType::kNormal}}, {"inplace_update_support", {offsetof(struct ColumnFamilyOptions, inplace_update_support), - OptionType::kBoolean}}, + OptionType::kBoolean, OptionVerificationType::kNormal}}, {"level_compaction_dynamic_level_bytes", {offsetof(struct ColumnFamilyOptions, level_compaction_dynamic_level_bytes), - OptionType::kBoolean}}, + OptionType::kBoolean, OptionVerificationType::kNormal}}, {"optimize_filters_for_hits", {offsetof(struct ColumnFamilyOptions, optimize_filters_for_hits), - OptionType::kBoolean}}, + OptionType::kBoolean, OptionVerificationType::kNormal}}, {"paranoid_file_checks", {offsetof(struct ColumnFamilyOptions, paranoid_file_checks), - OptionType::kBoolean}}, + OptionType::kBoolean, OptionVerificationType::kNormal}}, {"purge_redundant_kvs_while_flush", {offsetof(struct ColumnFamilyOptions, purge_redundant_kvs_while_flush), - OptionType::kBoolean}}, + OptionType::kBoolean, OptionVerificationType::kNormal}}, {"verify_checksums_in_compaction", {offsetof(struct ColumnFamilyOptions, verify_checksums_in_compaction), - OptionType::kBoolean}}, + OptionType::kBoolean, OptionVerificationType::kNormal}}, {"hard_pending_compaction_bytes_limit", {offsetof(struct ColumnFamilyOptions, hard_pending_compaction_bytes_limit), - OptionType::kUInt64T}}, + OptionType::kUInt64T, OptionVerificationType::kNormal}}, {"hard_rate_limit", {offsetof(struct ColumnFamilyOptions, hard_rate_limit), - OptionType::kDouble}}, + OptionType::kDouble, OptionVerificationType::kDeprecated}}, {"soft_rate_limit", {offsetof(struct ColumnFamilyOptions, soft_rate_limit), - OptionType::kDouble}}, + OptionType::kDouble, OptionVerificationType::kNormal}}, {"expanded_compaction_factor", {offsetof(struct ColumnFamilyOptions, expanded_compaction_factor), - OptionType::kInt}}, + OptionType::kInt, OptionVerificationType::kNormal}}, {"level0_file_num_compaction_trigger", {offsetof(struct ColumnFamilyOptions, level0_file_num_compaction_trigger), - OptionType::kInt}}, + OptionType::kInt, OptionVerificationType::kNormal}}, {"level0_slowdown_writes_trigger", {offsetof(struct ColumnFamilyOptions, level0_slowdown_writes_trigger), - OptionType::kInt}}, + OptionType::kInt, OptionVerificationType::kNormal}}, {"level0_stop_writes_trigger", {offsetof(struct ColumnFamilyOptions, level0_stop_writes_trigger), - OptionType::kInt}}, + OptionType::kInt, OptionVerificationType::kNormal}}, {"max_bytes_for_level_multiplier", {offsetof(struct ColumnFamilyOptions, max_bytes_for_level_multiplier), - OptionType::kInt}}, + OptionType::kInt, OptionVerificationType::kNormal}}, {"max_grandparent_overlap_factor", {offsetof(struct ColumnFamilyOptions, max_grandparent_overlap_factor), - OptionType::kInt}}, + OptionType::kInt, OptionVerificationType::kNormal}}, {"max_mem_compaction_level", {offsetof(struct ColumnFamilyOptions, max_mem_compaction_level), - OptionType::kInt}}, + OptionType::kInt, OptionVerificationType::kDeprecated}}, {"max_write_buffer_number", {offsetof(struct ColumnFamilyOptions, max_write_buffer_number), - OptionType::kInt}}, + OptionType::kInt, OptionVerificationType::kNormal}}, {"max_write_buffer_number_to_maintain", {offsetof(struct ColumnFamilyOptions, max_write_buffer_number_to_maintain), - OptionType::kInt}}, + OptionType::kInt, OptionVerificationType::kNormal}}, {"min_write_buffer_number_to_merge", {offsetof(struct ColumnFamilyOptions, min_write_buffer_number_to_merge), - OptionType::kInt}}, + OptionType::kInt, OptionVerificationType::kNormal}}, {"num_levels", - {offsetof(struct ColumnFamilyOptions, num_levels), OptionType::kInt}}, + {offsetof(struct ColumnFamilyOptions, num_levels), OptionType::kInt, + OptionVerificationType::kNormal}}, {"source_compaction_factor", {offsetof(struct ColumnFamilyOptions, source_compaction_factor), - OptionType::kInt}}, + OptionType::kInt, OptionVerificationType::kNormal}}, {"target_file_size_multiplier", {offsetof(struct ColumnFamilyOptions, target_file_size_multiplier), - OptionType::kInt}}, + OptionType::kInt, OptionVerificationType::kNormal}}, {"arena_block_size", {offsetof(struct ColumnFamilyOptions, arena_block_size), - OptionType::kSizeT}}, + OptionType::kSizeT, OptionVerificationType::kNormal}}, {"inplace_update_num_locks", {offsetof(struct ColumnFamilyOptions, inplace_update_num_locks), - OptionType::kSizeT}}, + OptionType::kSizeT, OptionVerificationType::kNormal}}, {"max_successive_merges", {offsetof(struct ColumnFamilyOptions, max_successive_merges), - OptionType::kSizeT}}, + OptionType::kSizeT, OptionVerificationType::kNormal}}, {"memtable_prefix_bloom_huge_page_tlb_size", {offsetof(struct ColumnFamilyOptions, memtable_prefix_bloom_huge_page_tlb_size), - OptionType::kSizeT}}, + OptionType::kSizeT, OptionVerificationType::kNormal}}, {"write_buffer_size", {offsetof(struct ColumnFamilyOptions, write_buffer_size), - OptionType::kSizeT}}, + OptionType::kSizeT, OptionVerificationType::kNormal}}, {"bloom_locality", {offsetof(struct ColumnFamilyOptions, bloom_locality), - OptionType::kUInt32T}}, + OptionType::kUInt32T, OptionVerificationType::kNormal}}, {"memtable_prefix_bloom_bits", {offsetof(struct ColumnFamilyOptions, memtable_prefix_bloom_bits), - OptionType::kUInt32T}}, + OptionType::kUInt32T, OptionVerificationType::kNormal}}, {"memtable_prefix_bloom_probes", {offsetof(struct ColumnFamilyOptions, memtable_prefix_bloom_probes), - OptionType::kUInt32T}}, + OptionType::kUInt32T, OptionVerificationType::kNormal}}, {"min_partial_merge_operands", {offsetof(struct ColumnFamilyOptions, min_partial_merge_operands), - OptionType::kUInt32T}}, + OptionType::kUInt32T, OptionVerificationType::kNormal}}, {"max_bytes_for_level_base", {offsetof(struct ColumnFamilyOptions, max_bytes_for_level_base), - OptionType::kUInt64T}}, + OptionType::kUInt64T, OptionVerificationType::kNormal}}, {"max_sequential_skip_in_iterations", {offsetof(struct ColumnFamilyOptions, max_sequential_skip_in_iterations), - OptionType::kUInt64T}}, + OptionType::kUInt64T, OptionVerificationType::kNormal}}, {"target_file_size_base", {offsetof(struct ColumnFamilyOptions, target_file_size_base), - OptionType::kUInt64T}}, + OptionType::kUInt64T, OptionVerificationType::kNormal}}, {"rate_limit_delay_max_milliseconds", {offsetof(struct ColumnFamilyOptions, rate_limit_delay_max_milliseconds), - OptionType::kUInt}}, + OptionType::kUInt, OptionVerificationType::kDeprecated}}, {"compaction_style", {offsetof(struct ColumnFamilyOptions, compaction_style), - OptionType::kCompactionStyle}}}; + OptionType::kCompactionStyle, OptionVerificationType::kNormal}}}; } // namespace rocksdb diff --git a/util/options_parser.cc b/util/options_parser.cc index b8052c3a6..be9bfc187 100644 --- a/util/options_parser.cc +++ b/util/options_parser.cc @@ -532,6 +532,11 @@ Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( Status RocksDBOptionsParser::VerifyDBOptions(const DBOptions& base_opt, const DBOptions& new_opt) { for (auto pair : db_options_type_info) { + if (pair.second.verification == OptionVerificationType::kDeprecated) { + // We skip checking deprecated variables as they might + // contain random values since they might not be initialized + continue; + } if (!AreEqualOptions(reinterpret_cast(&base_opt), reinterpret_cast(&new_opt), pair.second)) { @@ -547,6 +552,11 @@ Status RocksDBOptionsParser::VerifyDBOptions(const DBOptions& base_opt, Status RocksDBOptionsParser::VerifyCFOptions( const ColumnFamilyOptions& base_opt, const ColumnFamilyOptions& new_opt) { for (auto& pair : cf_options_type_info) { + if (pair.second.verification == OptionVerificationType::kDeprecated) { + // We skip checking deprecated variables as they might + // contain random values since they might not be initialized + continue; + } if (!AreEqualOptions(reinterpret_cast(&base_opt), reinterpret_cast(&new_opt), pair.second)) { -- GitLab