Better handling of deprecated options in RocksDBOptionsParser

Summary:
Previously, we treat deprecated options as normal options in
RocksDBOptionsParser.  However, these deprecated options should
not be verified and serialized.

Test Plan: options_test

Reviewers: igor, sdong, IslamAbdelRahman, anthony

Reviewed By: anthony

Subscribers: dhruba, leveldb

Differential Revision: https://reviews.facebook.net/D47775

util/options_helper.cc

@@ -702,6 +702,9 @@ Status GetStringFromColumnFamilyOptions(std::string* opt_string,
   opt_string->clear();
   for (auto iter = cf_options_type_info.begin();
        iter != cf_options_type_info.end(); ++iter) {
+    if (iter->second.verification == OptionVerificationType::kDeprecated) {
+      continue;
+    }
     std::string single_output;
     bool result = SerializeSingleColumnFamilyOption(&single_output, cf_options,
                                                     iter->first, delimiter);

util/options_helper.h

@@ -68,11 +68,21 @@ enum class OptionType {
   kUnknown
 };
 
+enum class OptionVerificationType {
+  kNormal,
+  kDeprecated  // The option is no longer used in rocksdb. The RocksDB
+               // OptionsParser will still accept this option if it
+               // happen to exists in some Options file.  However, the
+               // parser will not include it in serialization and
+               // verification processes.
+};
+
 // A struct for storing constant option information such as option name,
 // option type, and offset.
 struct OptionTypeInfo {
   int offset;
   OptionType type;
+  OptionVerificationType verification;
 };
 
 static std::unordered_map<std::string, OptionTypeInfo> db_options_type_info = {
@@ -91,94 +101,125 @@ static std::unordered_map<std::string, OptionTypeInfo> db_options_type_info = {
       std::vector<std::shared_ptr<EventListener>> listeners;
      */
     {"advise_random_on_open",
-     {offsetof(struct DBOptions, advise_random_on_open), OptionType::kBoolean}},
+     {offsetof(struct DBOptions, advise_random_on_open), OptionType::kBoolean,
+      OptionVerificationType::kNormal}},
     {"allow_mmap_reads",
-     {offsetof(struct DBOptions, allow_mmap_reads), OptionType::kBoolean}},
+     {offsetof(struct DBOptions, allow_mmap_reads), OptionType::kBoolean,
+      OptionVerificationType::kNormal}},
     {"allow_mmap_writes",
-     {offsetof(struct DBOptions, allow_mmap_writes), OptionType::kBoolean}},
+     {offsetof(struct DBOptions, allow_mmap_writes), OptionType::kBoolean,
+      OptionVerificationType::kNormal}},
     {"allow_os_buffer",
-     {offsetof(struct DBOptions, allow_os_buffer), OptionType::kBoolean}},
+     {offsetof(struct DBOptions, allow_os_buffer), OptionType::kBoolean,
+      OptionVerificationType::kNormal}},
     {"create_if_missing",
-     {offsetof(struct DBOptions, create_if_missing), OptionType::kBoolean}},
+     {offsetof(struct DBOptions, create_if_missing), OptionType::kBoolean,
+      OptionVerificationType::kNormal}},
     {"create_missing_column_families",
      {offsetof(struct DBOptions, create_missing_column_families),
-      OptionType::kBoolean}},
+      OptionType::kBoolean, OptionVerificationType::kNormal}},
     {"disableDataSync",
-     {offsetof(struct DBOptions, disableDataSync), OptionType::kBoolean}},
+     {offsetof(struct DBOptions, disableDataSync), OptionType::kBoolean,
+      OptionVerificationType::kNormal}},
     {"disable_data_sync",  // for compatibility
-     {offsetof(struct DBOptions, disableDataSync), OptionType::kBoolean}},
+     {offsetof(struct DBOptions, disableDataSync), OptionType::kBoolean,
+      OptionVerificationType::kNormal}},
     {"enable_thread_tracking",
-     {offsetof(struct DBOptions, enable_thread_tracking),
-      OptionType::kBoolean}},
+     {offsetof(struct DBOptions, enable_thread_tracking), OptionType::kBoolean,
+      OptionVerificationType::kNormal}},
     {"error_if_exists",
-     {offsetof(struct DBOptions, error_if_exists), OptionType::kBoolean}},
+     {offsetof(struct DBOptions, error_if_exists), OptionType::kBoolean,
+      OptionVerificationType::kNormal}},
     {"is_fd_close_on_exec",
-     {offsetof(struct DBOptions, is_fd_close_on_exec), OptionType::kBoolean}},
+     {offsetof(struct DBOptions, is_fd_close_on_exec), OptionType::kBoolean,
+      OptionVerificationType::kNormal}},
     {"paranoid_checks",
-     {offsetof(struct DBOptions, paranoid_checks), OptionType::kBoolean}},
+     {offsetof(struct DBOptions, paranoid_checks), OptionType::kBoolean,
+      OptionVerificationType::kNormal}},
     {"skip_log_error_on_recovery",
      {offsetof(struct DBOptions, skip_log_error_on_recovery),
-      OptionType::kBoolean}},
+      OptionType::kBoolean, OptionVerificationType::kNormal}},
     {"skip_stats_update_on_db_open",
      {offsetof(struct DBOptions, skip_stats_update_on_db_open),
-      OptionType::kBoolean}},
+      OptionType::kBoolean, OptionVerificationType::kNormal}},
     {"new_table_reader_for_compaction_inputs",
      {offsetof(struct DBOptions, new_table_reader_for_compaction_inputs),
-      OptionType::kBoolean}},
+      OptionType::kBoolean, OptionVerificationType::kNormal}},
     {"compaction_readahead_size",
-     {offsetof(struct DBOptions, compaction_readahead_size),
-      OptionType::kSizeT}},
+     {offsetof(struct DBOptions, compaction_readahead_size), OptionType::kSizeT,
+      OptionVerificationType::kNormal}},
     {"use_adaptive_mutex",
-     {offsetof(struct DBOptions, use_adaptive_mutex), OptionType::kBoolean}},
+     {offsetof(struct DBOptions, use_adaptive_mutex), OptionType::kBoolean,
+      OptionVerificationType::kNormal}},
     {"use_fsync",
-     {offsetof(struct DBOptions, use_fsync), OptionType::kBoolean}},
+     {offsetof(struct DBOptions, use_fsync), OptionType::kBoolean,
+      OptionVerificationType::kNormal}},
     {"max_background_compactions",
-     {offsetof(struct DBOptions, max_background_compactions),
-      OptionType::kInt}},
+     {offsetof(struct DBOptions, max_background_compactions), OptionType::kInt,
+      OptionVerificationType::kNormal}},
     {"max_background_flushes",
-     {offsetof(struct DBOptions, max_background_flushes), OptionType::kInt}},
+     {offsetof(struct DBOptions, max_background_flushes), OptionType::kInt,
+      OptionVerificationType::kNormal}},
     {"max_file_opening_threads",
-     {offsetof(struct DBOptions, max_file_opening_threads), OptionType::kInt}},
+     {offsetof(struct DBOptions, max_file_opening_threads), OptionType::kInt,
+      OptionVerificationType::kNormal}},
     {"max_open_files",
-     {offsetof(struct DBOptions, max_open_files), OptionType::kInt}},
+     {offsetof(struct DBOptions, max_open_files), OptionType::kInt,
+      OptionVerificationType::kNormal}},
     {"table_cache_numshardbits",
-     {offsetof(struct DBOptions, table_cache_numshardbits), OptionType::kInt}},
+     {offsetof(struct DBOptions, table_cache_numshardbits), OptionType::kInt,
+      OptionVerificationType::kNormal}},
     {"db_write_buffer_size",
-     {offsetof(struct DBOptions, db_write_buffer_size), OptionType::kSizeT}},
+     {offsetof(struct DBOptions, db_write_buffer_size), OptionType::kSizeT,
+      OptionVerificationType::kNormal}},
     {"keep_log_file_num",
-     {offsetof(struct DBOptions, keep_log_file_num), OptionType::kSizeT}},
+     {offsetof(struct DBOptions, keep_log_file_num), OptionType::kSizeT,
+      OptionVerificationType::kNormal}},
     {"log_file_time_to_roll",
-     {offsetof(struct DBOptions, log_file_time_to_roll), OptionType::kSizeT}},
+     {offsetof(struct DBOptions, log_file_time_to_roll), OptionType::kSizeT,
+      OptionVerificationType::kNormal}},
     {"manifest_preallocation_size",
      {offsetof(struct DBOptions, manifest_preallocation_size),
-      OptionType::kSizeT}},
+      OptionType::kSizeT, OptionVerificationType::kNormal}},
     {"max_log_file_size",
-     {offsetof(struct DBOptions, max_log_file_size), OptionType::kSizeT}},
+     {offsetof(struct DBOptions, max_log_file_size), OptionType::kSizeT,
+      OptionVerificationType::kNormal}},
     {"db_log_dir",
-     {offsetof(struct DBOptions, db_log_dir), OptionType::kString}},
-    {"wal_dir", {offsetof(struct DBOptions, wal_dir), OptionType::kString}},
+     {offsetof(struct DBOptions, db_log_dir), OptionType::kString,
+      OptionVerificationType::kNormal}},
+    {"wal_dir",
+     {offsetof(struct DBOptions, wal_dir), OptionType::kString,
+      OptionVerificationType::kNormal}},
     {"max_subcompactions",
-     {offsetof(struct DBOptions, max_subcompactions), OptionType::kUInt32T}},
+     {offsetof(struct DBOptions, max_subcompactions), OptionType::kUInt32T,
+      OptionVerificationType::kNormal}},
     {"WAL_size_limit_MB",
-     {offsetof(struct DBOptions, WAL_size_limit_MB), OptionType::kUInt64T}},
+     {offsetof(struct DBOptions, WAL_size_limit_MB), OptionType::kUInt64T,
+      OptionVerificationType::kNormal}},
     {"WAL_ttl_seconds",
-     {offsetof(struct DBOptions, WAL_ttl_seconds), OptionType::kUInt64T}},
+     {offsetof(struct DBOptions, WAL_ttl_seconds), OptionType::kUInt64T,
+      OptionVerificationType::kNormal}},
     {"bytes_per_sync",
-     {offsetof(struct DBOptions, bytes_per_sync), OptionType::kUInt64T}},
+     {offsetof(struct DBOptions, bytes_per_sync), OptionType::kUInt64T,
+      OptionVerificationType::kNormal}},
     {"delayed_write_rate",
-     {offsetof(struct DBOptions, delayed_write_rate), OptionType::kUInt64T}},
+     {offsetof(struct DBOptions, delayed_write_rate), OptionType::kUInt64T,
+      OptionVerificationType::kNormal}},
     {"delete_obsolete_files_period_micros",
      {offsetof(struct DBOptions, delete_obsolete_files_period_micros),
-      OptionType::kUInt64T}},
+      OptionType::kUInt64T, OptionVerificationType::kNormal}},
     {"max_manifest_file_size",
-     {offsetof(struct DBOptions, max_manifest_file_size),
-      OptionType::kUInt64T}},
+     {offsetof(struct DBOptions, max_manifest_file_size), OptionType::kUInt64T,
+      OptionVerificationType::kNormal}},
     {"max_total_wal_size",
-     {offsetof(struct DBOptions, max_total_wal_size), OptionType::kUInt64T}},
+     {offsetof(struct DBOptions, max_total_wal_size), OptionType::kUInt64T,
+      OptionVerificationType::kNormal}},
     {"wal_bytes_per_sync",
-     {offsetof(struct DBOptions, wal_bytes_per_sync), OptionType::kUInt64T}},
+     {offsetof(struct DBOptions, wal_bytes_per_sync), OptionType::kUInt64T,
+      OptionVerificationType::kNormal}},
     {"stats_dump_period_sec",
-     {offsetof(struct DBOptions, stats_dump_period_sec), OptionType::kUInt}}};
+     {offsetof(struct DBOptions, stats_dump_period_sec), OptionType::kUInt,
+      OptionVerificationType::kNormal}}};
 
 static std::unordered_map<std::string, OptionTypeInfo> cf_options_type_info = {
     /* not yet supported
@@ -205,122 +246,123 @@ static std::unordered_map<std::string, OptionTypeInfo> cf_options_type_info = {
      */
     {"compaction_measure_io_stats",
      {offsetof(struct ColumnFamilyOptions, compaction_measure_io_stats),
-      OptionType::kBoolean}},
+      OptionType::kBoolean, OptionVerificationType::kNormal}},
     {"disable_auto_compactions",
      {offsetof(struct ColumnFamilyOptions, disable_auto_compactions),
-      OptionType::kBoolean}},
+      OptionType::kBoolean, OptionVerificationType::kNormal}},
     {"filter_deletes",
      {offsetof(struct ColumnFamilyOptions, filter_deletes),
-      OptionType::kBoolean}},
+      OptionType::kBoolean, OptionVerificationType::kNormal}},
     {"inplace_update_support",
      {offsetof(struct ColumnFamilyOptions, inplace_update_support),
-      OptionType::kBoolean}},
+      OptionType::kBoolean, OptionVerificationType::kNormal}},
     {"level_compaction_dynamic_level_bytes",
      {offsetof(struct ColumnFamilyOptions,
                level_compaction_dynamic_level_bytes),
-      OptionType::kBoolean}},
+      OptionType::kBoolean, OptionVerificationType::kNormal}},
     {"optimize_filters_for_hits",
      {offsetof(struct ColumnFamilyOptions, optimize_filters_for_hits),
-      OptionType::kBoolean}},
+      OptionType::kBoolean, OptionVerificationType::kNormal}},
     {"paranoid_file_checks",
      {offsetof(struct ColumnFamilyOptions, paranoid_file_checks),
-      OptionType::kBoolean}},
+      OptionType::kBoolean, OptionVerificationType::kNormal}},
     {"purge_redundant_kvs_while_flush",
      {offsetof(struct ColumnFamilyOptions, purge_redundant_kvs_while_flush),
-      OptionType::kBoolean}},
+      OptionType::kBoolean, OptionVerificationType::kNormal}},
     {"verify_checksums_in_compaction",
      {offsetof(struct ColumnFamilyOptions, verify_checksums_in_compaction),
-      OptionType::kBoolean}},
+      OptionType::kBoolean, OptionVerificationType::kNormal}},
     {"hard_pending_compaction_bytes_limit",
      {offsetof(struct ColumnFamilyOptions, hard_pending_compaction_bytes_limit),
-      OptionType::kUInt64T}},
+      OptionType::kUInt64T, OptionVerificationType::kNormal}},
     {"hard_rate_limit",
      {offsetof(struct ColumnFamilyOptions, hard_rate_limit),
-      OptionType::kDouble}},
+      OptionType::kDouble, OptionVerificationType::kDeprecated}},
     {"soft_rate_limit",
      {offsetof(struct ColumnFamilyOptions, soft_rate_limit),
-      OptionType::kDouble}},
+      OptionType::kDouble, OptionVerificationType::kNormal}},
     {"expanded_compaction_factor",
      {offsetof(struct ColumnFamilyOptions, expanded_compaction_factor),
-      OptionType::kInt}},
+      OptionType::kInt, OptionVerificationType::kNormal}},
     {"level0_file_num_compaction_trigger",
      {offsetof(struct ColumnFamilyOptions, level0_file_num_compaction_trigger),
-      OptionType::kInt}},
+      OptionType::kInt, OptionVerificationType::kNormal}},
     {"level0_slowdown_writes_trigger",
      {offsetof(struct ColumnFamilyOptions, level0_slowdown_writes_trigger),
-      OptionType::kInt}},
+      OptionType::kInt, OptionVerificationType::kNormal}},
     {"level0_stop_writes_trigger",
      {offsetof(struct ColumnFamilyOptions, level0_stop_writes_trigger),
-      OptionType::kInt}},
+      OptionType::kInt, OptionVerificationType::kNormal}},
     {"max_bytes_for_level_multiplier",
      {offsetof(struct ColumnFamilyOptions, max_bytes_for_level_multiplier),
-      OptionType::kInt}},
+      OptionType::kInt, OptionVerificationType::kNormal}},
     {"max_grandparent_overlap_factor",
      {offsetof(struct ColumnFamilyOptions, max_grandparent_overlap_factor),
-      OptionType::kInt}},
+      OptionType::kInt, OptionVerificationType::kNormal}},
     {"max_mem_compaction_level",
      {offsetof(struct ColumnFamilyOptions, max_mem_compaction_level),
-      OptionType::kInt}},
+      OptionType::kInt, OptionVerificationType::kDeprecated}},
     {"max_write_buffer_number",
      {offsetof(struct ColumnFamilyOptions, max_write_buffer_number),
-      OptionType::kInt}},
+      OptionType::kInt, OptionVerificationType::kNormal}},
     {"max_write_buffer_number_to_maintain",
      {offsetof(struct ColumnFamilyOptions, max_write_buffer_number_to_maintain),
-      OptionType::kInt}},
+      OptionType::kInt, OptionVerificationType::kNormal}},
     {"min_write_buffer_number_to_merge",
      {offsetof(struct ColumnFamilyOptions, min_write_buffer_number_to_merge),
-      OptionType::kInt}},
+      OptionType::kInt, OptionVerificationType::kNormal}},
     {"num_levels",
-     {offsetof(struct ColumnFamilyOptions, num_levels), OptionType::kInt}},
+     {offsetof(struct ColumnFamilyOptions, num_levels), OptionType::kInt,
+      OptionVerificationType::kNormal}},
     {"source_compaction_factor",
      {offsetof(struct ColumnFamilyOptions, source_compaction_factor),
-      OptionType::kInt}},
+      OptionType::kInt, OptionVerificationType::kNormal}},
     {"target_file_size_multiplier",
      {offsetof(struct ColumnFamilyOptions, target_file_size_multiplier),
-      OptionType::kInt}},
+      OptionType::kInt, OptionVerificationType::kNormal}},
     {"arena_block_size",
      {offsetof(struct ColumnFamilyOptions, arena_block_size),
-      OptionType::kSizeT}},
+      OptionType::kSizeT, OptionVerificationType::kNormal}},
     {"inplace_update_num_locks",
      {offsetof(struct ColumnFamilyOptions, inplace_update_num_locks),
-      OptionType::kSizeT}},
+      OptionType::kSizeT, OptionVerificationType::kNormal}},
     {"max_successive_merges",
      {offsetof(struct ColumnFamilyOptions, max_successive_merges),
-      OptionType::kSizeT}},
+      OptionType::kSizeT, OptionVerificationType::kNormal}},
     {"memtable_prefix_bloom_huge_page_tlb_size",
      {offsetof(struct ColumnFamilyOptions,
                memtable_prefix_bloom_huge_page_tlb_size),
-      OptionType::kSizeT}},
+      OptionType::kSizeT, OptionVerificationType::kNormal}},
     {"write_buffer_size",
      {offsetof(struct ColumnFamilyOptions, write_buffer_size),
-      OptionType::kSizeT}},
+      OptionType::kSizeT, OptionVerificationType::kNormal}},
     {"bloom_locality",
      {offsetof(struct ColumnFamilyOptions, bloom_locality),
-      OptionType::kUInt32T}},
+      OptionType::kUInt32T, OptionVerificationType::kNormal}},
     {"memtable_prefix_bloom_bits",
      {offsetof(struct ColumnFamilyOptions, memtable_prefix_bloom_bits),
-      OptionType::kUInt32T}},
+      OptionType::kUInt32T, OptionVerificationType::kNormal}},
     {"memtable_prefix_bloom_probes",
      {offsetof(struct ColumnFamilyOptions, memtable_prefix_bloom_probes),
-      OptionType::kUInt32T}},
+      OptionType::kUInt32T, OptionVerificationType::kNormal}},
     {"min_partial_merge_operands",
      {offsetof(struct ColumnFamilyOptions, min_partial_merge_operands),
-      OptionType::kUInt32T}},
+      OptionType::kUInt32T, OptionVerificationType::kNormal}},
     {"max_bytes_for_level_base",
      {offsetof(struct ColumnFamilyOptions, max_bytes_for_level_base),
-      OptionType::kUInt64T}},
+      OptionType::kUInt64T, OptionVerificationType::kNormal}},
     {"max_sequential_skip_in_iterations",
      {offsetof(struct ColumnFamilyOptions, max_sequential_skip_in_iterations),
-      OptionType::kUInt64T}},
+      OptionType::kUInt64T, OptionVerificationType::kNormal}},
     {"target_file_size_base",
      {offsetof(struct ColumnFamilyOptions, target_file_size_base),
-      OptionType::kUInt64T}},
+      OptionType::kUInt64T, OptionVerificationType::kNormal}},
     {"rate_limit_delay_max_milliseconds",
      {offsetof(struct ColumnFamilyOptions, rate_limit_delay_max_milliseconds),
-      OptionType::kUInt}},
+      OptionType::kUInt, OptionVerificationType::kDeprecated}},
     {"compaction_style",
      {offsetof(struct ColumnFamilyOptions, compaction_style),
-      OptionType::kCompactionStyle}}};
+      OptionType::kCompactionStyle, OptionVerificationType::kNormal}}};
 
 }  // namespace rocksdb
 

util/options_parser.cc

@@ -532,6 +532,11 @@ Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
 Status RocksDBOptionsParser::VerifyDBOptions(const DBOptions& base_opt,
                                              const DBOptions& new_opt) {
   for (auto pair : db_options_type_info) {
+    if (pair.second.verification == OptionVerificationType::kDeprecated) {
+      // We skip checking deprecated variables as they might
+      // contain random values since they might not be initialized
+      continue;
+    }
     if (!AreEqualOptions(reinterpret_cast<const char*>(&base_opt),
                          reinterpret_cast<const char*>(&new_opt),
                          pair.second)) {
@@ -547,6 +552,11 @@ Status RocksDBOptionsParser::VerifyDBOptions(const DBOptions& base_opt,
 Status RocksDBOptionsParser::VerifyCFOptions(
     const ColumnFamilyOptions& base_opt, const ColumnFamilyOptions& new_opt) {
   for (auto& pair : cf_options_type_info) {
+    if (pair.second.verification == OptionVerificationType::kDeprecated) {
+      // We skip checking deprecated variables as they might
+      // contain random values since they might not be initialized
+      continue;
+    }
     if (!AreEqualOptions(reinterpret_cast<const char*>(&base_opt),
                          reinterpret_cast<const char*>(&new_opt),
                          pair.second)) {