diff --git a/include/rocksdb/convenience.h b/include/rocksdb/convenience.h index 1c1057d3a03051e9ba5fcde74ad4fc3654d4f840..1b2f3eec675a35ebe93d9f22451a6dbf65fea028 100644 --- a/include/rocksdb/convenience.h +++ b/include/rocksdb/convenience.h @@ -48,6 +48,9 @@ Status GetDBOptionsFromString( const std::string& opts_str, DBOptions* new_options); +Status GetStringFromDBOptions(const DBOptions& db_options, + std::string* opts_str); + Status GetBlockBasedTableOptionsFromString( const BlockBasedTableOptions& table_options, const std::string& opts_str, diff --git a/util/options_helper.cc b/util/options_helper.cc index d1b4e346e080bb9749b80986df504d763f15d6fc..9afaf317d32dccf2b2d7d7dabe6662ede46a1151 100644 --- a/util/options_helper.cc +++ b/util/options_helper.cc @@ -17,6 +17,7 @@ #include "rocksdb/table.h" #include "table/block_based_table_factory.h" #include "util/logging.h" +#include "util/string_util.h" namespace rocksdb { @@ -483,82 +484,103 @@ bool ParseColumnFamilyOption(const std::string& name, const std::string& value, return true; } +bool SerializeSingleDBOption(const DBOptions& db_options, + const std::string& name, std::string* opt_string) { + auto iter = db_options_type_info.find(name); + if (iter == db_options_type_info.end()) { + return false; + } + auto& opt_info = iter->second; + const char* opt_address = + reinterpret_cast(&db_options) + opt_info.offset; + std::string value; + + switch (opt_info.type) { + case OptionType::kBoolean: + value = *(reinterpret_cast(opt_address)) ? "true" : "false"; + break; + case OptionType::kInt: + value = ToString(*(reinterpret_cast(opt_address))); + break; + case OptionType::kUInt: + value = ToString(*(reinterpret_cast(opt_address))); + break; + case OptionType::kUInt32T: + value = ToString(*(reinterpret_cast(opt_address))); + break; + case OptionType::kUInt64T: + value = ToString(*(reinterpret_cast(opt_address))); + break; + case OptionType::kSizeT: + value = ToString(*(reinterpret_cast(opt_address))); + break; + case OptionType::kString: + value = *(reinterpret_cast(opt_address)); + break; + default: + return false; + } + + *opt_string = name + " = " + value + "; "; + return true; +} + +Status GetStringFromDBOptions(const DBOptions& db_options, + std::string* opt_string) { + assert(opt_string); + opt_string->clear(); + for (auto iter = db_options_type_info.begin(); + iter != db_options_type_info.end(); ++iter) { + std::string single_output; + bool result = + SerializeSingleDBOption(db_options, iter->first, &single_output); + assert(result); + if (result) { + opt_string->append(single_output); + } + } + return Status::OK(); +} + bool ParseDBOption(const std::string& name, const std::string& value, DBOptions* new_options) { try { - if (name == "create_if_missing") { - new_options->create_if_missing = ParseBoolean(name, value); - } else if (name == "create_missing_column_families") { - new_options->create_missing_column_families = - ParseBoolean(name, value); - } else if (name == "error_if_exists") { - new_options->error_if_exists = ParseBoolean(name, value); - } else if (name == "paranoid_checks") { - new_options->paranoid_checks = ParseBoolean(name, value); - } else if (name == "rate_limiter_bytes_per_sec") { + if (name == "rate_limiter_bytes_per_sec") { new_options->rate_limiter.reset( NewGenericRateLimiter(static_cast(ParseUint64(value)))); - } else if (name == "max_open_files") { - new_options->max_open_files = ParseInt(value); - } else if (name == "max_total_wal_size") { - new_options->max_total_wal_size = ParseUint64(value); - } else if (name == "disable_data_sync") { - new_options->disableDataSync = ParseBoolean(name, value); - } else if (name == "use_fsync") { - new_options->use_fsync = ParseBoolean(name, value); - } else if (name == "db_paths") { - // TODO(ljin): add support - return false; - } else if (name == "db_log_dir") { - new_options->db_log_dir = value; - } else if (name == "wal_dir") { - new_options->wal_dir = value; - } else if (name == "delete_obsolete_files_period_micros") { - new_options->delete_obsolete_files_period_micros = ParseUint64(value); - } else if (name == "max_background_compactions") { - new_options->max_background_compactions = ParseInt(value); - } else if (name == "max_background_flushes") { - new_options->max_background_flushes = ParseInt(value); - } else if (name == "max_log_file_size") { - new_options->max_log_file_size = ParseSizeT(value); - } else if (name == "log_file_time_to_roll") { - new_options->log_file_time_to_roll = ParseSizeT(value); - } else if (name == "keep_log_file_num") { - new_options->keep_log_file_num = ParseSizeT(value); - } else if (name == "max_manifest_file_size") { - new_options->max_manifest_file_size = ParseUint64(value); - } else if (name == "table_cache_numshardbits") { - new_options->table_cache_numshardbits = ParseInt(value); - } else if (name == "WAL_ttl_seconds") { - new_options->WAL_ttl_seconds = ParseUint64(value); - } else if (name == "WAL_size_limit_MB") { - new_options->WAL_size_limit_MB = ParseUint64(value); - } else if (name == "manifest_preallocation_size") { - new_options->manifest_preallocation_size = ParseSizeT(value); - } else if (name == "allow_os_buffer") { - new_options->allow_os_buffer = ParseBoolean(name, value); - } else if (name == "allow_mmap_reads") { - new_options->allow_mmap_reads = ParseBoolean(name, value); - } else if (name == "allow_mmap_writes") { - new_options->allow_mmap_writes = ParseBoolean(name, value); - } else if (name == "is_fd_close_on_exec") { - new_options->is_fd_close_on_exec = ParseBoolean(name, value); - } else if (name == "skip_log_error_on_recovery") { - new_options->skip_log_error_on_recovery = ParseBoolean(name, value); - } else if (name == "stats_dump_period_sec") { - new_options->stats_dump_period_sec = ParseUint32(value); - } else if (name == "advise_random_on_open") { - new_options->advise_random_on_open = ParseBoolean(name, value); - } else if (name == "db_write_buffer_size") { - new_options->db_write_buffer_size = ParseUint64(value); - } else if (name == "use_adaptive_mutex") { - new_options->use_adaptive_mutex = ParseBoolean(name, value); - } else if (name == "bytes_per_sync") { - new_options->bytes_per_sync = ParseUint64(value); - } else if (name == "wal_bytes_per_sync") { - new_options->wal_bytes_per_sync = ParseUint64(value); } else { - return false; + auto iter = db_options_type_info.find(name); + if (iter == db_options_type_info.end()) { + return false; + } + auto& opt_info = iter->second; + char* opt_address = + reinterpret_cast(new_options) + opt_info.offset; + switch (opt_info.type) { + case OptionType::kBoolean: + *reinterpret_cast(opt_address) = ParseBoolean("", value); + break; + case OptionType::kInt: + *reinterpret_cast(opt_address) = ParseInt(value); + break; + case OptionType::kUInt: + *reinterpret_cast(opt_address) = ParseUint32(value); + break; + case OptionType::kUInt32T: + *reinterpret_cast(opt_address) = ParseUint32(value); + break; + case OptionType::kUInt64T: + *reinterpret_cast(opt_address) = ParseUint64(value); + break; + case OptionType::kSizeT: + *reinterpret_cast(opt_address) = ParseUint32(value); + break; + case OptionType::kString: + *reinterpret_cast(opt_address) = value; + break; + default: + return false; + } } } catch (const std::exception& e) { return false; diff --git a/util/options_helper.h b/util/options_helper.h index 02c78811441b9ae6867e7914e31d25ffd97f37b5..79299d79cc0b60597cca4b9281f9579898681a05 100644 --- a/util/options_helper.h +++ b/util/options_helper.h @@ -17,4 +17,121 @@ Status GetMutableOptionsFromStrings( const std::unordered_map& options_map, MutableCFOptions* new_options); +enum class OptionType { + kBoolean, + kInt, + kUInt, + kUInt32T, + kUInt64T, + kSizeT, + kString, + kUnknown +}; + +// A struct for storing constant option information such as option name, +// option type, and offset. +struct OptionTypeInfo { + int offset; + OptionType type; +}; + +static std::unordered_map db_options_type_info = { + /* + // not yet supported + AccessHint access_hint_on_compaction_start; + Env* env; + InfoLogLevel info_log_level; + WALRecoveryMode wal_recovery_mode; + std::shared_ptr row_cache; + std::shared_ptr delete_scheduler; + std::shared_ptr info_log; + std::shared_ptr rate_limiter; + std::shared_ptr statistics; + std::vector db_paths; + std::vector> listeners; + */ + {"advise_random_on_open", + {offsetof(struct DBOptions, advise_random_on_open), OptionType::kBoolean}}, + {"allow_mmap_reads", + {offsetof(struct DBOptions, allow_mmap_reads), OptionType::kBoolean}}, + {"allow_mmap_writes", + {offsetof(struct DBOptions, allow_mmap_writes), OptionType::kBoolean}}, + {"allow_os_buffer", + {offsetof(struct DBOptions, allow_os_buffer), OptionType::kBoolean}}, + {"create_if_missing", + {offsetof(struct DBOptions, create_if_missing), OptionType::kBoolean}}, + {"create_missing_column_families", + {offsetof(struct DBOptions, create_missing_column_families), + OptionType::kBoolean}}, + {"disableDataSync", + {offsetof(struct DBOptions, disableDataSync), OptionType::kBoolean}}, + {"disable_data_sync", // for compatibility + {offsetof(struct DBOptions, disableDataSync), OptionType::kBoolean}}, + {"enable_thread_tracking", + {offsetof(struct DBOptions, enable_thread_tracking), + OptionType::kBoolean}}, + {"error_if_exists", + {offsetof(struct DBOptions, error_if_exists), OptionType::kBoolean}}, + {"is_fd_close_on_exec", + {offsetof(struct DBOptions, is_fd_close_on_exec), OptionType::kBoolean}}, + {"paranoid_checks", + {offsetof(struct DBOptions, paranoid_checks), OptionType::kBoolean}}, + {"skip_log_error_on_recovery", + {offsetof(struct DBOptions, skip_log_error_on_recovery), + OptionType::kBoolean}}, + {"skip_stats_update_on_db_open", + {offsetof(struct DBOptions, skip_stats_update_on_db_open), + OptionType::kBoolean}}, + {"use_adaptive_mutex", + {offsetof(struct DBOptions, use_adaptive_mutex), OptionType::kBoolean}}, + {"use_fsync", + {offsetof(struct DBOptions, use_fsync), OptionType::kBoolean}}, + {"max_background_compactions", + {offsetof(struct DBOptions, max_background_compactions), + OptionType::kInt}}, + {"max_background_flushes", + {offsetof(struct DBOptions, max_background_flushes), OptionType::kInt}}, + {"max_file_opening_threads", + {offsetof(struct DBOptions, max_file_opening_threads), OptionType::kInt}}, + {"max_open_files", + {offsetof(struct DBOptions, max_open_files), OptionType::kInt}}, + {"table_cache_numshardbits", + {offsetof(struct DBOptions, table_cache_numshardbits), OptionType::kInt}}, + {"db_write_buffer_size", + {offsetof(struct DBOptions, db_write_buffer_size), OptionType::kSizeT}}, + {"keep_log_file_num", + {offsetof(struct DBOptions, keep_log_file_num), OptionType::kSizeT}}, + {"log_file_time_to_roll", + {offsetof(struct DBOptions, log_file_time_to_roll), OptionType::kSizeT}}, + {"manifest_preallocation_size", + {offsetof(struct DBOptions, manifest_preallocation_size), + OptionType::kSizeT}}, + {"max_log_file_size", + {offsetof(struct DBOptions, max_log_file_size), OptionType::kSizeT}}, + {"db_log_dir", + {offsetof(struct DBOptions, db_log_dir), OptionType::kString}}, + {"wal_dir", {offsetof(struct DBOptions, wal_dir), OptionType::kString}}, + {"num_subcompactions", + {offsetof(struct DBOptions, num_subcompactions), OptionType::kUInt32T}}, + {"WAL_size_limit_MB", + {offsetof(struct DBOptions, WAL_size_limit_MB), OptionType::kUInt64T}}, + {"WAL_ttl_seconds", + {offsetof(struct DBOptions, WAL_ttl_seconds), OptionType::kUInt64T}}, + {"bytes_per_sync", + {offsetof(struct DBOptions, bytes_per_sync), OptionType::kUInt64T}}, + {"delayed_write_rate", + {offsetof(struct DBOptions, delayed_write_rate), OptionType::kUInt64T}}, + {"delete_obsolete_files_period_micros", + {offsetof(struct DBOptions, delete_obsolete_files_period_micros), + OptionType::kUInt64T}}, + {"max_manifest_file_size", + {offsetof(struct DBOptions, max_manifest_file_size), + OptionType::kUInt64T}}, + {"max_total_wal_size", + {offsetof(struct DBOptions, max_total_wal_size), OptionType::kUInt64T}}, + {"wal_bytes_per_sync", + {offsetof(struct DBOptions, wal_bytes_per_sync), OptionType::kUInt64T}}, + {"stats_dump_period_sec", + {offsetof(struct DBOptions, stats_dump_period_sec), OptionType::kUInt}}}; + } // namespace rocksdb diff --git a/util/options_test.cc b/util/options_test.cc index 0b2cce883a8db755c7200e3cc8ffb693f750394c..fbf428b9f5c3d52e82f12075857579789b5a0775 100644 --- a/util/options_test.cc +++ b/util/options_test.cc @@ -504,6 +504,137 @@ TEST_F(OptionsTest, GetOptionsFromStringTest) { ASSERT_EQ(new_options.max_open_files, 1); ASSERT_TRUE(new_options.rate_limiter.get() != nullptr); } + +namespace { +void VerifyDBOptions(const DBOptions& base_opt, const DBOptions& new_opt) { + // boolean options + ASSERT_EQ(base_opt.advise_random_on_open, new_opt.advise_random_on_open); + ASSERT_EQ(base_opt.allow_mmap_reads, new_opt.allow_mmap_reads); + ASSERT_EQ(base_opt.allow_mmap_writes, new_opt.allow_mmap_writes); + ASSERT_EQ(base_opt.allow_os_buffer, new_opt.allow_os_buffer); + ASSERT_EQ(base_opt.create_if_missing, new_opt.create_if_missing); + ASSERT_EQ(base_opt.create_missing_column_families, + new_opt.create_missing_column_families); + ASSERT_EQ(base_opt.disableDataSync, new_opt.disableDataSync); + ASSERT_EQ(base_opt.enable_thread_tracking, new_opt.enable_thread_tracking); + ASSERT_EQ(base_opt.error_if_exists, new_opt.error_if_exists); + ASSERT_EQ(base_opt.is_fd_close_on_exec, new_opt.is_fd_close_on_exec); + ASSERT_EQ(base_opt.paranoid_checks, new_opt.paranoid_checks); + ASSERT_EQ(base_opt.skip_log_error_on_recovery, + new_opt.skip_log_error_on_recovery); + ASSERT_EQ(base_opt.skip_stats_update_on_db_open, + new_opt.skip_stats_update_on_db_open); + ASSERT_EQ(base_opt.use_adaptive_mutex, new_opt.use_adaptive_mutex); + ASSERT_EQ(base_opt.use_fsync, new_opt.use_fsync); + + // int options + ASSERT_EQ(base_opt.max_background_compactions, + new_opt.max_background_compactions); + ASSERT_EQ(base_opt.max_background_flushes, new_opt.max_background_flushes); + ASSERT_EQ(base_opt.max_file_opening_threads, + new_opt.max_file_opening_threads); + ASSERT_EQ(base_opt.max_open_files, new_opt.max_open_files); + ASSERT_EQ(base_opt.table_cache_numshardbits, + new_opt.table_cache_numshardbits); + + // size_t options + ASSERT_EQ(base_opt.db_write_buffer_size, new_opt.db_write_buffer_size); + ASSERT_EQ(base_opt.keep_log_file_num, new_opt.keep_log_file_num); + ASSERT_EQ(base_opt.log_file_time_to_roll, new_opt.log_file_time_to_roll); + ASSERT_EQ(base_opt.manifest_preallocation_size, + new_opt.manifest_preallocation_size); + ASSERT_EQ(base_opt.max_log_file_size, new_opt.max_log_file_size); + + // std::string options + ASSERT_EQ(base_opt.db_log_dir, new_opt.db_log_dir); + ASSERT_EQ(base_opt.wal_dir, new_opt.wal_dir); + + // uint32_t options + ASSERT_EQ(base_opt.num_subcompactions, new_opt.num_subcompactions); + + // uint64_t options + ASSERT_EQ(base_opt.WAL_size_limit_MB, new_opt.WAL_size_limit_MB); + ASSERT_EQ(base_opt.WAL_ttl_seconds, new_opt.WAL_ttl_seconds); + ASSERT_EQ(base_opt.bytes_per_sync, new_opt.bytes_per_sync); + ASSERT_EQ(base_opt.delayed_write_rate, new_opt.delayed_write_rate); + ASSERT_EQ(base_opt.delete_obsolete_files_period_micros, + new_opt.delete_obsolete_files_period_micros); + ASSERT_EQ(base_opt.max_manifest_file_size, new_opt.max_manifest_file_size); + ASSERT_EQ(base_opt.max_total_wal_size, new_opt.max_total_wal_size); + ASSERT_EQ(base_opt.wal_bytes_per_sync, new_opt.wal_bytes_per_sync); + + // unsigned int options + ASSERT_EQ(base_opt.stats_dump_period_sec, new_opt.stats_dump_period_sec); +} +} // namespace + +TEST_F(OptionsTest, DBOptionsSerialization) { + Options base_options, new_options; + Random rnd(301); + + // Phase 1: Make big change in base_options + // boolean options + base_options.advise_random_on_open = rnd.Uniform(2); + base_options.allow_mmap_reads = rnd.Uniform(2); + base_options.allow_mmap_writes = rnd.Uniform(2); + base_options.allow_os_buffer = rnd.Uniform(2); + base_options.create_if_missing = rnd.Uniform(2); + base_options.create_missing_column_families = rnd.Uniform(2); + base_options.disableDataSync = rnd.Uniform(2); + base_options.enable_thread_tracking = rnd.Uniform(2); + base_options.error_if_exists = rnd.Uniform(2); + base_options.is_fd_close_on_exec = rnd.Uniform(2); + base_options.paranoid_checks = rnd.Uniform(2); + base_options.skip_log_error_on_recovery = rnd.Uniform(2); + base_options.skip_stats_update_on_db_open = rnd.Uniform(2); + base_options.use_adaptive_mutex = rnd.Uniform(2); + base_options.use_fsync = rnd.Uniform(2); + + // int options + base_options.max_background_compactions = rnd.Uniform(100); + base_options.max_background_flushes = rnd.Uniform(100); + base_options.max_file_opening_threads = rnd.Uniform(100); + base_options.max_open_files = rnd.Uniform(100); + base_options.table_cache_numshardbits = rnd.Uniform(100); + + // size_t options + base_options.db_write_buffer_size = rnd.Uniform(10000); + base_options.keep_log_file_num = rnd.Uniform(10000); + base_options.log_file_time_to_roll = rnd.Uniform(10000); + base_options.manifest_preallocation_size = rnd.Uniform(10000); + base_options.max_log_file_size = rnd.Uniform(10000); + + // std::string options + base_options.db_log_dir = "path/to/db_log_dir"; + base_options.wal_dir = "path/to/wal_dir"; + + // uint32_t options + base_options.num_subcompactions = rnd.Uniform(100000); + + // uint64_t options + static const uint64_t uint_max = static_cast(UINT_MAX); + base_options.WAL_size_limit_MB = uint_max + rnd.Uniform(100000); + base_options.WAL_ttl_seconds = uint_max + rnd.Uniform(100000); + base_options.bytes_per_sync = uint_max + rnd.Uniform(100000); + base_options.delayed_write_rate = uint_max + rnd.Uniform(100000); + base_options.delete_obsolete_files_period_micros = + uint_max + rnd.Uniform(100000); + base_options.max_manifest_file_size = uint_max + rnd.Uniform(100000); + base_options.max_total_wal_size = uint_max + rnd.Uniform(100000); + base_options.wal_bytes_per_sync = uint_max + rnd.Uniform(100000); + + // unsigned int options + base_options.stats_dump_period_sec = rnd.Uniform(100000); + + // Phase 2: obtain a string from base_option + std::string base_opt_string; + ASSERT_OK(GetStringFromDBOptions(base_options, &base_opt_string)); + + // Phase 3: Set new_options from the derived string and expect + // new_options == base_options + ASSERT_OK(GetDBOptionsFromString(DBOptions(), base_opt_string, &new_options)); + VerifyDBOptions(base_options, new_options); +} #endif // !ROCKSDB_LITE