options_helper.cc 58.2 KB
Newer Older
1
//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
S
Siying Dong 已提交
2 3 4
//  This source code is licensed under both the GPLv2 (found in the
//  COPYING file in the root directory) and Apache 2.0 License
//  (found in the LICENSE.Apache file in the root directory).
5
#include "options/options_helper.h"
6 7

#include <cassert>
L
Lei Jin 已提交
8
#include <cctype>
S
sdong 已提交
9
#include <cstdlib>
10
#include <unordered_set>
11
#include <vector>
12

13
#include "rocksdb/cache.h"
14
#include "rocksdb/compaction_filter.h"
A
agiardullo 已提交
15
#include "rocksdb/convenience.h"
16
#include "rocksdb/filter_policy.h"
17 18
#include "rocksdb/memtablerep.h"
#include "rocksdb/merge_operator.h"
19
#include "rocksdb/options.h"
I
Igor Canadi 已提交
20
#include "rocksdb/rate_limiter.h"
21
#include "rocksdb/slice_transform.h"
22
#include "rocksdb/table.h"
23
#include "rocksdb/utilities/object_registry.h"
24
#include "table/block_based/block_based_table_factory.h"
25
#include "table/plain/plain_table_factory.h"
26
#include "util/string_util.h"
27

28
namespace ROCKSDB_NAMESPACE {
29

30 31 32 33 34 35 36 37 38 39 40 41 42 43
DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options,
                         const MutableDBOptions& mutable_db_options) {
  DBOptions options;

  options.create_if_missing = immutable_db_options.create_if_missing;
  options.create_missing_column_families =
      immutable_db_options.create_missing_column_families;
  options.error_if_exists = immutable_db_options.error_if_exists;
  options.paranoid_checks = immutable_db_options.paranoid_checks;
  options.env = immutable_db_options.env;
  options.rate_limiter = immutable_db_options.rate_limiter;
  options.sst_file_manager = immutable_db_options.sst_file_manager;
  options.info_log = immutable_db_options.info_log;
  options.info_log_level = immutable_db_options.info_log_level;
L
Leonidas Galanis 已提交
44
  options.max_open_files = mutable_db_options.max_open_files;
45 46
  options.max_file_opening_threads =
      immutable_db_options.max_file_opening_threads;
47
  options.max_total_wal_size = mutable_db_options.max_total_wal_size;
48 49 50 51 52 53
  options.statistics = immutable_db_options.statistics;
  options.use_fsync = immutable_db_options.use_fsync;
  options.db_paths = immutable_db_options.db_paths;
  options.db_log_dir = immutable_db_options.db_log_dir;
  options.wal_dir = immutable_db_options.wal_dir;
  options.delete_obsolete_files_period_micros =
54
      mutable_db_options.delete_obsolete_files_period_micros;
55
  options.max_background_jobs = mutable_db_options.max_background_jobs;
56
  options.base_background_compactions =
57
      mutable_db_options.base_background_compactions;
58
  options.max_background_compactions =
59
      mutable_db_options.max_background_compactions;
60 61
  options.bytes_per_sync = mutable_db_options.bytes_per_sync;
  options.wal_bytes_per_sync = mutable_db_options.wal_bytes_per_sync;
62
  options.strict_bytes_per_sync = mutable_db_options.strict_bytes_per_sync;
63
  options.max_subcompactions = immutable_db_options.max_subcompactions;
64
  options.max_background_flushes = mutable_db_options.max_background_flushes;
65 66 67 68 69 70 71 72 73 74 75 76 77
  options.max_log_file_size = immutable_db_options.max_log_file_size;
  options.log_file_time_to_roll = immutable_db_options.log_file_time_to_roll;
  options.keep_log_file_num = immutable_db_options.keep_log_file_num;
  options.recycle_log_file_num = immutable_db_options.recycle_log_file_num;
  options.max_manifest_file_size = immutable_db_options.max_manifest_file_size;
  options.table_cache_numshardbits =
      immutable_db_options.table_cache_numshardbits;
  options.WAL_ttl_seconds = immutable_db_options.wal_ttl_seconds;
  options.WAL_size_limit_MB = immutable_db_options.wal_size_limit_mb;
  options.manifest_preallocation_size =
      immutable_db_options.manifest_preallocation_size;
  options.allow_mmap_reads = immutable_db_options.allow_mmap_reads;
  options.allow_mmap_writes = immutable_db_options.allow_mmap_writes;
78
  options.use_direct_reads = immutable_db_options.use_direct_reads;
79 80
  options.use_direct_io_for_flush_and_compaction =
      immutable_db_options.use_direct_io_for_flush_and_compaction;
81 82
  options.allow_fallocate = immutable_db_options.allow_fallocate;
  options.is_fd_close_on_exec = immutable_db_options.is_fd_close_on_exec;
83
  options.stats_dump_period_sec = mutable_db_options.stats_dump_period_sec;
84 85
  options.stats_persist_period_sec =
      mutable_db_options.stats_persist_period_sec;
86
  options.persist_stats_to_disk = immutable_db_options.persist_stats_to_disk;
87 88
  options.stats_history_buffer_size =
      mutable_db_options.stats_history_buffer_size;
89 90 91 92 93 94 95 96
  options.advise_random_on_open = immutable_db_options.advise_random_on_open;
  options.db_write_buffer_size = immutable_db_options.db_write_buffer_size;
  options.write_buffer_manager = immutable_db_options.write_buffer_manager;
  options.access_hint_on_compaction_start =
      immutable_db_options.access_hint_on_compaction_start;
  options.new_table_reader_for_compaction_inputs =
      immutable_db_options.new_table_reader_for_compaction_inputs;
  options.compaction_readahead_size =
97
      mutable_db_options.compaction_readahead_size;
98 99 100
  options.random_access_max_buffer_size =
      immutable_db_options.random_access_max_buffer_size;
  options.writable_file_max_buffer_size =
101
      mutable_db_options.writable_file_max_buffer_size;
102 103 104
  options.use_adaptive_mutex = immutable_db_options.use_adaptive_mutex;
  options.listeners = immutable_db_options.listeners;
  options.enable_thread_tracking = immutable_db_options.enable_thread_tracking;
105
  options.delayed_write_rate = mutable_db_options.delayed_write_rate;
106
  options.enable_pipelined_write = immutable_db_options.enable_pipelined_write;
M
Maysam Yabandeh 已提交
107
  options.unordered_write = immutable_db_options.unordered_write;
108 109 110 111
  options.allow_concurrent_memtable_write =
      immutable_db_options.allow_concurrent_memtable_write;
  options.enable_write_thread_adaptive_yield =
      immutable_db_options.enable_write_thread_adaptive_yield;
112 113
  options.max_write_batch_group_size_bytes =
      immutable_db_options.max_write_batch_group_size_bytes;
114 115 116 117 118 119
  options.write_thread_max_yield_usec =
      immutable_db_options.write_thread_max_yield_usec;
  options.write_thread_slow_yield_usec =
      immutable_db_options.write_thread_slow_yield_usec;
  options.skip_stats_update_on_db_open =
      immutable_db_options.skip_stats_update_on_db_open;
120 121
  options.skip_checking_sst_file_sizes_on_db_open =
      immutable_db_options.skip_checking_sst_file_sizes_on_db_open;
122 123 124 125 126 127 128 129 130 131 132
  options.wal_recovery_mode = immutable_db_options.wal_recovery_mode;
  options.allow_2pc = immutable_db_options.allow_2pc;
  options.row_cache = immutable_db_options.row_cache;
#ifndef ROCKSDB_LITE
  options.wal_filter = immutable_db_options.wal_filter;
#endif  // ROCKSDB_LITE
  options.fail_if_options_file_error =
      immutable_db_options.fail_if_options_file_error;
  options.dump_malloc_stats = immutable_db_options.dump_malloc_stats;
  options.avoid_flush_during_recovery =
      immutable_db_options.avoid_flush_during_recovery;
Y
Yi Wu 已提交
133 134
  options.avoid_flush_during_shutdown =
      mutable_db_options.avoid_flush_during_shutdown;
135 136
  options.allow_ingest_behind =
      immutable_db_options.allow_ingest_behind;
137 138
  options.preserve_deletes =
      immutable_db_options.preserve_deletes;
139 140
  options.two_write_queues = immutable_db_options.two_write_queues;
  options.manual_wal_flush = immutable_db_options.manual_wal_flush;
141
  options.atomic_flush = immutable_db_options.atomic_flush;
142 143
  options.avoid_unnecessary_blocking_io =
      immutable_db_options.avoid_unnecessary_blocking_io;
144
  options.log_readahead_size = immutable_db_options.log_readahead_size;
145 146
  options.file_checksum_gen_factory =
      immutable_db_options.file_checksum_gen_factory;
147
  options.best_efforts_recovery = immutable_db_options.best_efforts_recovery;
148 149 150
  return options;
}

151 152 153 154 155 156 157 158 159 160 161
ColumnFamilyOptions BuildColumnFamilyOptions(
    const ColumnFamilyOptions& options,
    const MutableCFOptions& mutable_cf_options) {
  ColumnFamilyOptions cf_opts(options);

  // Memtable related options
  cf_opts.write_buffer_size = mutable_cf_options.write_buffer_size;
  cf_opts.max_write_buffer_number = mutable_cf_options.max_write_buffer_number;
  cf_opts.arena_block_size = mutable_cf_options.arena_block_size;
  cf_opts.memtable_prefix_bloom_size_ratio =
      mutable_cf_options.memtable_prefix_bloom_size_ratio;
162 163
  cf_opts.memtable_whole_key_filtering =
      mutable_cf_options.memtable_whole_key_filtering;
164 165 166 167
  cf_opts.memtable_huge_page_size = mutable_cf_options.memtable_huge_page_size;
  cf_opts.max_successive_merges = mutable_cf_options.max_successive_merges;
  cf_opts.inplace_update_num_locks =
      mutable_cf_options.inplace_update_num_locks;
168
  cf_opts.prefix_extractor = mutable_cf_options.prefix_extractor;
169 170 171 172

  // Compaction related options
  cf_opts.disable_auto_compactions =
      mutable_cf_options.disable_auto_compactions;
173 174 175 176
  cf_opts.soft_pending_compaction_bytes_limit =
      mutable_cf_options.soft_pending_compaction_bytes_limit;
  cf_opts.hard_pending_compaction_bytes_limit =
      mutable_cf_options.hard_pending_compaction_bytes_limit;
177 178 179 180 181 182 183 184 185 186 187 188 189 190
  cf_opts.level0_file_num_compaction_trigger =
      mutable_cf_options.level0_file_num_compaction_trigger;
  cf_opts.level0_slowdown_writes_trigger =
      mutable_cf_options.level0_slowdown_writes_trigger;
  cf_opts.level0_stop_writes_trigger =
      mutable_cf_options.level0_stop_writes_trigger;
  cf_opts.max_compaction_bytes = mutable_cf_options.max_compaction_bytes;
  cf_opts.target_file_size_base = mutable_cf_options.target_file_size_base;
  cf_opts.target_file_size_multiplier =
      mutable_cf_options.target_file_size_multiplier;
  cf_opts.max_bytes_for_level_base =
      mutable_cf_options.max_bytes_for_level_base;
  cf_opts.max_bytes_for_level_multiplier =
      mutable_cf_options.max_bytes_for_level_multiplier;
191
  cf_opts.ttl = mutable_cf_options.ttl;
S
Sagar Vemuri 已提交
192 193
  cf_opts.periodic_compaction_seconds =
      mutable_cf_options.periodic_compaction_seconds;
194 195 196 197 198 199 200

  cf_opts.max_bytes_for_level_multiplier_additional.clear();
  for (auto value :
       mutable_cf_options.max_bytes_for_level_multiplier_additional) {
    cf_opts.max_bytes_for_level_multiplier_additional.emplace_back(value);
  }

201
  cf_opts.compaction_options_fifo = mutable_cf_options.compaction_options_fifo;
202 203
  cf_opts.compaction_options_universal =
      mutable_cf_options.compaction_options_universal;
204

205 206 207 208 209 210
  // Misc options
  cf_opts.max_sequential_skip_in_iterations =
      mutable_cf_options.max_sequential_skip_in_iterations;
  cf_opts.paranoid_file_checks = mutable_cf_options.paranoid_file_checks;
  cf_opts.report_bg_io_stats = mutable_cf_options.report_bg_io_stats;
  cf_opts.compression = mutable_cf_options.compression;
211 212 213 214
  cf_opts.compression_opts = mutable_cf_options.compression_opts;
  cf_opts.bottommost_compression = mutable_cf_options.bottommost_compression;
  cf_opts.bottommost_compression_opts =
      mutable_cf_options.bottommost_compression_opts;
215
  cf_opts.sample_for_compression = mutable_cf_options.sample_for_compression;
216 217 218 219 220 221 222 223

  cf_opts.table_factory = options.table_factory;
  // TODO(yhchiang): find some way to handle the following derived options
  // * max_file_size

  return cf_opts;
}

224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244
std::map<CompactionStyle, std::string>
    OptionsHelper::compaction_style_to_string = {
        {kCompactionStyleLevel, "kCompactionStyleLevel"},
        {kCompactionStyleUniversal, "kCompactionStyleUniversal"},
        {kCompactionStyleFIFO, "kCompactionStyleFIFO"},
        {kCompactionStyleNone, "kCompactionStyleNone"}};

std::map<CompactionPri, std::string> OptionsHelper::compaction_pri_to_string = {
    {kByCompensatedSize, "kByCompensatedSize"},
    {kOldestLargestSeqFirst, "kOldestLargestSeqFirst"},
    {kOldestSmallestSeqFirst, "kOldestSmallestSeqFirst"},
    {kMinOverlappingRatio, "kMinOverlappingRatio"}};

std::map<CompactionStopStyle, std::string>
    OptionsHelper::compaction_stop_style_to_string = {
        {kCompactionStopStyleSimilarSize, "kCompactionStopStyleSimilarSize"},
        {kCompactionStopStyleTotalSize, "kCompactionStopStyleTotalSize"}};

std::unordered_map<std::string, ChecksumType>
    OptionsHelper::checksum_type_string_map = {{"kNoChecksum", kNoChecksum},
                                               {"kCRC32c", kCRC32c},
B
Bo Hou 已提交
245 246
                                               {"kxxHash", kxxHash},
                                               {"kxxHash64", kxxHash64}};
247

248 249 250 251 252 253 254 255 256 257 258 259
std::unordered_map<std::string, CompressionType>
    OptionsHelper::compression_type_string_map = {
        {"kNoCompression", kNoCompression},
        {"kSnappyCompression", kSnappyCompression},
        {"kZlibCompression", kZlibCompression},
        {"kBZip2Compression", kBZip2Compression},
        {"kLZ4Compression", kLZ4Compression},
        {"kLZ4HCCompression", kLZ4HCCompression},
        {"kXpressCompression", kXpressCompression},
        {"kZSTD", kZSTD},
        {"kZSTDNotFinalCompression", kZSTDNotFinalCompression},
        {"kDisableCompressionOption", kDisableCompressionOption}};
260
#ifndef ROCKSDB_LITE
261

262
const std::string kNameEnv = "env";
263 264
const std::string kOptNameBMCompOpts = "bottommost_compression_opts";
const std::string kOptNameCompOpts = "compression_opts";
265

D
Dmitri Smirnov 已提交
266
namespace {
267 268
template <typename T>
bool ParseEnum(const std::unordered_map<std::string, T>& type_map,
S
SherlockNoMad 已提交
269 270 271 272 273 274 275 276 277
               const std::string& type, T* value) {
  auto iter = type_map.find(type);
  if (iter != type_map.end()) {
    *value = iter->second;
    return true;
  }
  return false;
}

278 279
template <typename T>
bool SerializeEnum(const std::unordered_map<std::string, T>& type_map,
S
SherlockNoMad 已提交
280 281 282 283
                   const T& type, std::string* value) {
  for (const auto& pair : type_map) {
    if (pair.second == type) {
      *value = pair.first;
284
      return true;
S
SherlockNoMad 已提交
285
    }
286
  }
S
SherlockNoMad 已提交
287
  return false;
288 289 290 291 292 293 294 295 296 297 298
}

bool SerializeVectorCompressionType(const std::vector<CompressionType>& types,
                                    std::string* value) {
  std::stringstream ss;
  bool result;
  for (size_t i = 0; i < types.size(); ++i) {
    if (i > 0) {
      ss << ':';
    }
    std::string string_type;
S
SherlockNoMad 已提交
299
    result = SerializeEnum<CompressionType>(compression_type_string_map,
300
                                            types[i], &string_type);
301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319
    if (result == false) {
      return result;
    }
    ss << string_type;
  }
  *value = ss.str();
  return true;
}

bool ParseVectorCompressionType(
    const std::string& value,
    std::vector<CompressionType>* compression_per_level) {
  compression_per_level->clear();
  size_t start = 0;
  while (start < value.size()) {
    size_t end = value.find(':', start);
    bool is_ok;
    CompressionType type;
    if (end == std::string::npos) {
320 321
      is_ok = ParseEnum<CompressionType>(compression_type_string_map,
                                         value.substr(start), &type);
322 323 324 325 326 327
      if (!is_ok) {
        return false;
      }
      compression_per_level->emplace_back(type);
      break;
    } else {
328 329
      is_ok = ParseEnum<CompressionType>(
          compression_type_string_map, value.substr(start, end - start), &type);
330 331 332 333 334 335 336 337 338 339
      if (!is_ok) {
        return false;
      }
      compression_per_level->emplace_back(type);
      start = end + 1;
    }
  }
  return true;
}

340 341 342 343 344 345 346 347 348 349 350 351 352 353 354
// This is to handle backward compatibility, where compaction_options_fifo
// could be assigned a single scalar value, say, like "23", which would be
// assigned to max_table_files_size.
bool FIFOCompactionOptionsSpecialCase(const std::string& opt_str,
                                      CompactionOptionsFIFO* options) {
  if (opt_str.find("=") != std::string::npos) {
    // New format. Go do your new parsing using ParseStructOptions.
    return false;
  }

  // Old format. Parse just a single uint64_t value.
  options->max_table_files_size = ParseUint64(opt_str);
  return true;
}

355 356
static bool SerializeStruct(
    const void* const options, std::string* value,
357
    const std::unordered_map<std::string, OptionTypeInfo>& type_info_map) {
358 359 360 361 362 363 364 365 366
  std::string opt_str;
  Status s = GetStringFromStruct(&opt_str, options, type_info_map, ";");
  if (!s.ok()) {
    return false;
  }
  *value = "{" + opt_str + "}";
  return true;
}

367 368
static bool ParseSingleStructOption(
    const std::string& opt_val_str, void* options,
369
    const std::unordered_map<std::string, OptionTypeInfo>& type_info_map) {
370 371 372 373 374 375 376 377
  size_t end = opt_val_str.find('=');
  std::string key = opt_val_str.substr(0, end);
  std::string value = opt_val_str.substr(end + 1);
  auto iter = type_info_map.find(key);
  if (iter == type_info_map.end()) {
    return false;
  }
  const auto& opt_info = iter->second;
378
  if (opt_info.IsDeprecated()) {
379 380 381 382
    // Should also skip deprecated sub-options such as
    // fifo_compaction_options_type_info.ttl
    return true;
  }
383 384 385 386 387
  return ParseOptionHelper(
      reinterpret_cast<char*>(options) + opt_info.mutable_offset, opt_info.type,
      value);
}

388 389
static bool ParseStructOptions(
    const std::string& opt_str, void* options,
390
    const std::unordered_map<std::string, OptionTypeInfo>& type_info_map) {
391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410
  assert(!opt_str.empty());

  size_t start = 0;
  if (opt_str[0] == '{') {
    start++;
  }
  while ((start != std::string::npos) && (start < opt_str.size())) {
    if (opt_str[start] == '}') {
      break;
    }
    size_t end = opt_str.find(';', start);
    size_t len = (end == std::string::npos) ? end : end - start;
    if (!ParseSingleStructOption(opt_str.substr(start, len), options,
                                 type_info_map)) {
      return false;
    }
    start = (end == std::string::npos) ? end : end + 1;
  }
  return true;
}
411
}  // anonymouse namespace
412

413 414 415 416
bool ParseSliceTransformHelper(
    const std::string& kFixedPrefixName, const std::string& kCappedPrefixName,
    const std::string& value,
    std::shared_ptr<const SliceTransform>* slice_transform) {
417 418
  const char* no_op_name = "rocksdb.Noop";
  size_t no_op_length = strlen(no_op_name);
419 420 421 422 423 424 425 426 427 428 429
  auto& pe_value = value;
  if (pe_value.size() > kFixedPrefixName.size() &&
      pe_value.compare(0, kFixedPrefixName.size(), kFixedPrefixName) == 0) {
    int prefix_length = ParseInt(trim(value.substr(kFixedPrefixName.size())));
    slice_transform->reset(NewFixedPrefixTransform(prefix_length));
  } else if (pe_value.size() > kCappedPrefixName.size() &&
             pe_value.compare(0, kCappedPrefixName.size(), kCappedPrefixName) ==
                 0) {
    int prefix_length =
        ParseInt(trim(pe_value.substr(kCappedPrefixName.size())));
    slice_transform->reset(NewCappedPrefixTransform(prefix_length));
430 431 432 433
  } else if (pe_value.size() == no_op_length &&
             pe_value.compare(0, no_op_length, no_op_name) == 0) {
    const SliceTransform* no_op_transform = NewNoopTransform();
    slice_transform->reset(no_op_transform);
434
  } else if (value == kNullptrString) {
435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467
    slice_transform->reset();
  } else {
    return false;
  }

  return true;
}

bool ParseSliceTransform(
    const std::string& value,
    std::shared_ptr<const SliceTransform>* slice_transform) {
  // While we normally don't convert the string representation of a
  // pointer-typed option into its instance, here we do so for backward
  // compatibility as we allow this action in SetOption().

  // TODO(yhchiang): A possible better place for these serialization /
  // deserialization is inside the class definition of pointer-typed
  // option itself, but this requires a bigger change of public API.
  bool result =
      ParseSliceTransformHelper("fixed:", "capped:", value, slice_transform);
  if (result) {
    return result;
  }
  result = ParseSliceTransformHelper(
      "rocksdb.FixedPrefix.", "rocksdb.CappedPrefix.", value, slice_transform);
  if (result) {
    return result;
  }
  // TODO(yhchiang): we can further support other default
  //                 SliceTransforms here.
  return false;
}

468 469 470 471 472 473 474 475 476
bool ParseOptionHelper(char* opt_address, const OptionType& opt_type,
                       const std::string& value) {
  switch (opt_type) {
    case OptionType::kBoolean:
      *reinterpret_cast<bool*>(opt_address) = ParseBoolean("", value);
      break;
    case OptionType::kInt:
      *reinterpret_cast<int*>(opt_address) = ParseInt(value);
      break;
Z
Zhongyi Xie 已提交
477 478 479 480 481 482
    case OptionType::kInt32T:
      *reinterpret_cast<int32_t*>(opt_address) = ParseInt32(value);
      break;
    case OptionType::kInt64T:
      PutUnaligned(reinterpret_cast<int64_t*>(opt_address), ParseInt64(value));
      break;
Y
Yi Wu 已提交
483 484 485
    case OptionType::kVectorInt:
      *reinterpret_cast<std::vector<int>*>(opt_address) = ParseVectorInt(value);
      break;
486 487 488 489 490 491 492
    case OptionType::kUInt:
      *reinterpret_cast<unsigned int*>(opt_address) = ParseUint32(value);
      break;
    case OptionType::kUInt32T:
      *reinterpret_cast<uint32_t*>(opt_address) = ParseUint32(value);
      break;
    case OptionType::kUInt64T:
T
Tomas Kolda 已提交
493
      PutUnaligned(reinterpret_cast<uint64_t*>(opt_address), ParseUint64(value));
494 495
      break;
    case OptionType::kSizeT:
T
Tomas Kolda 已提交
496
      PutUnaligned(reinterpret_cast<size_t*>(opt_address), ParseSizeT(value));
497 498 499 500 501 502 503 504
      break;
    case OptionType::kString:
      *reinterpret_cast<std::string*>(opt_address) = value;
      break;
    case OptionType::kDouble:
      *reinterpret_cast<double*>(opt_address) = ParseDouble(value);
      break;
    case OptionType::kCompactionStyle:
505 506 507
      return ParseEnum<CompactionStyle>(
          compaction_style_string_map, value,
          reinterpret_cast<CompactionStyle*>(opt_address));
508 509 510 511
    case OptionType::kCompactionPri:
      return ParseEnum<CompactionPri>(
          compaction_pri_string_map, value,
          reinterpret_cast<CompactionPri*>(opt_address));
512
    case OptionType::kCompressionType:
513 514 515
      return ParseEnum<CompressionType>(
          compression_type_string_map, value,
          reinterpret_cast<CompressionType*>(opt_address));
516 517 518 519 520 521 522
    case OptionType::kVectorCompressionType:
      return ParseVectorCompressionType(
          value, reinterpret_cast<std::vector<CompressionType>*>(opt_address));
    case OptionType::kSliceTransform:
      return ParseSliceTransform(
          value, reinterpret_cast<std::shared_ptr<const SliceTransform>*>(
                     opt_address));
523
    case OptionType::kChecksumType:
524 525 526
      return ParseEnum<ChecksumType>(
          checksum_type_string_map, value,
          reinterpret_cast<ChecksumType*>(opt_address));
527
    case OptionType::kBlockBasedTableIndexType:
S
SherlockNoMad 已提交
528
      return ParseEnum<BlockBasedTableOptions::IndexType>(
529 530
          block_base_table_index_type_string_map, value,
          reinterpret_cast<BlockBasedTableOptions::IndexType*>(opt_address));
531 532 533 534 535
    case OptionType::kBlockBasedTableDataBlockIndexType:
      return ParseEnum<BlockBasedTableOptions::DataBlockIndexType>(
          block_base_table_data_block_index_type_string_map, value,
          reinterpret_cast<BlockBasedTableOptions::DataBlockIndexType*>(
              opt_address));
536 537
    case OptionType::kBlockBasedTableIndexShorteningMode:
      return ParseEnum<BlockBasedTableOptions::IndexShorteningMode>(
538 539 540
          block_base_table_index_shortening_mode_string_map, value,
          reinterpret_cast<BlockBasedTableOptions::IndexShorteningMode*>(
              opt_address));
541
    case OptionType::kEncodingType:
542 543 544
      return ParseEnum<EncodingType>(
          encoding_type_string_map, value,
          reinterpret_cast<EncodingType*>(opt_address));
545 546 547 548 549 550 551 552 553 554 555 556
    case OptionType::kWALRecoveryMode:
      return ParseEnum<WALRecoveryMode>(
          wal_recovery_mode_string_map, value,
          reinterpret_cast<WALRecoveryMode*>(opt_address));
    case OptionType::kAccessHint:
      return ParseEnum<DBOptions::AccessHint>(
          access_hint_string_map, value,
          reinterpret_cast<DBOptions::AccessHint*>(opt_address));
    case OptionType::kInfoLogLevel:
      return ParseEnum<InfoLogLevel>(
          info_log_level_string_map, value,
          reinterpret_cast<InfoLogLevel*>(opt_address));
557 558 559
    case OptionType::kCompactionOptionsFIFO: {
      if (!FIFOCompactionOptionsSpecialCase(
              value, reinterpret_cast<CompactionOptionsFIFO*>(opt_address))) {
560 561
        return ParseStructOptions(value, opt_address,
                                  fifo_compaction_options_type_info);
562 563 564
      }
      return true;
    }
565
    case OptionType::kLRUCacheOptions: {
566 567
      return ParseStructOptions(value, opt_address,
                                lru_cache_options_type_info);
568
    }
569
    case OptionType::kCompactionOptionsUniversal:
570 571
      return ParseStructOptions(value, opt_address,
                                universal_compaction_options_type_info);
572 573 574 575
    case OptionType::kCompactionStopStyle:
      return ParseEnum<CompactionStopStyle>(
          compaction_stop_style_string_map, value,
          reinterpret_cast<CompactionStopStyle*>(opt_address));
576 577 578 579 580 581 582 583 584
    default:
      return false;
  }
  return true;
}

bool SerializeSingleOptionHelper(const char* opt_address,
                                 const OptionType opt_type,
                                 std::string* value) {
585

586 587 588 589 590 591 592 593
  assert(value);
  switch (opt_type) {
    case OptionType::kBoolean:
      *value = *(reinterpret_cast<const bool*>(opt_address)) ? "true" : "false";
      break;
    case OptionType::kInt:
      *value = ToString(*(reinterpret_cast<const int*>(opt_address)));
      break;
Z
Zhongyi Xie 已提交
594 595 596 597 598 599 600 601 602 603
    case OptionType::kInt32T:
      *value = ToString(*(reinterpret_cast<const int32_t*>(opt_address)));
      break;
    case OptionType::kInt64T:
      {
        int64_t v;
        GetUnaligned(reinterpret_cast<const int64_t*>(opt_address), &v);
        *value = ToString(v);
      }
      break;
Y
Yi Wu 已提交
604 605 606
    case OptionType::kVectorInt:
      return SerializeIntVector(
          *reinterpret_cast<const std::vector<int>*>(opt_address), value);
607 608 609 610 611 612 613
    case OptionType::kUInt:
      *value = ToString(*(reinterpret_cast<const unsigned int*>(opt_address)));
      break;
    case OptionType::kUInt32T:
      *value = ToString(*(reinterpret_cast<const uint32_t*>(opt_address)));
      break;
    case OptionType::kUInt64T:
T
Tomas Kolda 已提交
614 615 616 617 618
      {
        uint64_t v;
        GetUnaligned(reinterpret_cast<const uint64_t*>(opt_address), &v);
        *value = ToString(v);
      }
619 620
      break;
    case OptionType::kSizeT:
T
Tomas Kolda 已提交
621 622 623 624 625
      {
        size_t v;
        GetUnaligned(reinterpret_cast<const size_t*>(opt_address), &v);
        *value = ToString(v);
      }
626 627 628 629 630
      break;
    case OptionType::kDouble:
      *value = ToString(*(reinterpret_cast<const double*>(opt_address)));
      break;
    case OptionType::kString:
631 632
      *value = EscapeOptionString(
          *(reinterpret_cast<const std::string*>(opt_address)));
633 634
      break;
    case OptionType::kCompactionStyle:
635 636
      return SerializeEnum<CompactionStyle>(
          compaction_style_string_map,
S
SherlockNoMad 已提交
637
          *(reinterpret_cast<const CompactionStyle*>(opt_address)), value);
638 639 640 641
    case OptionType::kCompactionPri:
      return SerializeEnum<CompactionPri>(
          compaction_pri_string_map,
          *(reinterpret_cast<const CompactionPri*>(opt_address)), value);
642
    case OptionType::kCompressionType:
643 644
      return SerializeEnum<CompressionType>(
          compression_type_string_map,
645 646 647 648 649 650 651 652 653 654 655
          *(reinterpret_cast<const CompressionType*>(opt_address)), value);
    case OptionType::kVectorCompressionType:
      return SerializeVectorCompressionType(
          *(reinterpret_cast<const std::vector<CompressionType>*>(opt_address)),
          value);
      break;
    case OptionType::kSliceTransform: {
      const auto* slice_transform_ptr =
          reinterpret_cast<const std::shared_ptr<const SliceTransform>*>(
              opt_address);
      *value = slice_transform_ptr->get() ? slice_transform_ptr->get()->Name()
656
                                          : kNullptrString;
657 658 659 660 661 662 663
      break;
    }
    case OptionType::kTableFactory: {
      const auto* table_factory_ptr =
          reinterpret_cast<const std::shared_ptr<const TableFactory>*>(
              opt_address);
      *value = table_factory_ptr->get() ? table_factory_ptr->get()->Name()
664
                                        : kNullptrString;
665 666 667 668 669
      break;
    }
    case OptionType::kComparator: {
      // it's a const pointer of const Comparator*
      const auto* ptr = reinterpret_cast<const Comparator* const*>(opt_address);
670 671 672
      // Since the user-specified comparator will be wrapped by
      // InternalKeyComparator, we should persist the user-specified one
      // instead of InternalKeyComparator.
S
Siying Dong 已提交
673 674
      if (*ptr == nullptr) {
        *value = kNullptrString;
675
      } else {
S
Siying Dong 已提交
676 677 678 679 680
        const Comparator* root_comp = (*ptr)->GetRootComparator();
        if (root_comp == nullptr) {
          root_comp = (*ptr);
        }
        *value = root_comp->Name();
681
      }
682 683 684 685 686 687
      break;
    }
    case OptionType::kCompactionFilter: {
      // it's a const pointer of const CompactionFilter*
      const auto* ptr =
          reinterpret_cast<const CompactionFilter* const*>(opt_address);
688
      *value = *ptr ? (*ptr)->Name() : kNullptrString;
689 690 691 692 693 694
      break;
    }
    case OptionType::kCompactionFilterFactory: {
      const auto* ptr =
          reinterpret_cast<const std::shared_ptr<CompactionFilterFactory>*>(
              opt_address);
695
      *value = ptr->get() ? ptr->get()->Name() : kNullptrString;
696 697 698 699 700 701
      break;
    }
    case OptionType::kMemTableRepFactory: {
      const auto* ptr =
          reinterpret_cast<const std::shared_ptr<MemTableRepFactory>*>(
              opt_address);
702
      *value = ptr->get() ? ptr->get()->Name() : kNullptrString;
703 704 705 706 707
      break;
    }
    case OptionType::kMergeOperator: {
      const auto* ptr =
          reinterpret_cast<const std::shared_ptr<MergeOperator>*>(opt_address);
708 709 710 711 712 713 714 715 716 717
      *value = ptr->get() ? ptr->get()->Name() : kNullptrString;
      break;
    }
    case OptionType::kFilterPolicy: {
      const auto* ptr =
          reinterpret_cast<const std::shared_ptr<FilterPolicy>*>(opt_address);
      *value = ptr->get() ? ptr->get()->Name() : kNullptrString;
      break;
    }
    case OptionType::kChecksumType:
718 719
      return SerializeEnum<ChecksumType>(
          checksum_type_string_map,
720 721
          *reinterpret_cast<const ChecksumType*>(opt_address), value);
    case OptionType::kBlockBasedTableIndexType:
S
SherlockNoMad 已提交
722 723
      return SerializeEnum<BlockBasedTableOptions::IndexType>(
          block_base_table_index_type_string_map,
724 725 726
          *reinterpret_cast<const BlockBasedTableOptions::IndexType*>(
              opt_address),
          value);
727 728 729 730 731 732
    case OptionType::kBlockBasedTableDataBlockIndexType:
      return SerializeEnum<BlockBasedTableOptions::DataBlockIndexType>(
          block_base_table_data_block_index_type_string_map,
          *reinterpret_cast<const BlockBasedTableOptions::DataBlockIndexType*>(
              opt_address),
          value);
733 734 735 736 737 738
    case OptionType::kBlockBasedTableIndexShorteningMode:
      return SerializeEnum<BlockBasedTableOptions::IndexShorteningMode>(
          block_base_table_index_shortening_mode_string_map,
          *reinterpret_cast<const BlockBasedTableOptions::IndexShorteningMode*>(
              opt_address),
          value);
739 740 741 742 743
    case OptionType::kFlushBlockPolicyFactory: {
      const auto* ptr =
          reinterpret_cast<const std::shared_ptr<FlushBlockPolicyFactory>*>(
              opt_address);
      *value = ptr->get() ? ptr->get()->Name() : kNullptrString;
744 745
      break;
    }
746
    case OptionType::kEncodingType:
747 748
      return SerializeEnum<EncodingType>(
          encoding_type_string_map,
749
          *reinterpret_cast<const EncodingType*>(opt_address), value);
750 751 752 753 754 755 756 757 758 759 760 761
    case OptionType::kWALRecoveryMode:
      return SerializeEnum<WALRecoveryMode>(
          wal_recovery_mode_string_map,
          *reinterpret_cast<const WALRecoveryMode*>(opt_address), value);
    case OptionType::kAccessHint:
      return SerializeEnum<DBOptions::AccessHint>(
          access_hint_string_map,
          *reinterpret_cast<const DBOptions::AccessHint*>(opt_address), value);
    case OptionType::kInfoLogLevel:
      return SerializeEnum<InfoLogLevel>(
          info_log_level_string_map,
          *reinterpret_cast<const InfoLogLevel*>(opt_address), value);
762
    case OptionType::kCompactionOptionsFIFO:
763 764
      return SerializeStruct(opt_address, value,
                             fifo_compaction_options_type_info);
765
    case OptionType::kCompactionOptionsUniversal:
766 767
      return SerializeStruct(opt_address, value,
                             universal_compaction_options_type_info);
768 769 770 771
    case OptionType::kCompactionStopStyle:
      return SerializeEnum<CompactionStopStyle>(
          compaction_stop_style_string_map,
          *reinterpret_cast<const CompactionStopStyle*>(opt_address), value);
772 773 774 775 776 777
    default:
      return false;
  }
  return true;
}

778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824
Status ParseCompressionOptions(const std::string& value,
                               const std::string& name,
                               CompressionOptions& compression_opts) {
  size_t start = 0;
  size_t end = value.find(':');
  if (end == std::string::npos) {
    return Status::InvalidArgument("unable to parse the specified CF option " +
                                   name);
  }
  compression_opts.window_bits = ParseInt(value.substr(start, end - start));
  start = end + 1;
  end = value.find(':', start);
  if (end == std::string::npos) {
    return Status::InvalidArgument("unable to parse the specified CF option " +
                                   name);
  }
  compression_opts.level = ParseInt(value.substr(start, end - start));
  start = end + 1;
  if (start >= value.size()) {
    return Status::InvalidArgument("unable to parse the specified CF option " +
                                   name);
  }
  end = value.find(':', start);
  compression_opts.strategy =
      ParseInt(value.substr(start, value.size() - start));
  // max_dict_bytes is optional for backwards compatibility
  if (end != std::string::npos) {
    start = end + 1;
    if (start >= value.size()) {
      return Status::InvalidArgument(
          "unable to parse the specified CF option " + name);
    }
    compression_opts.max_dict_bytes =
        ParseInt(value.substr(start, value.size() - start));
    end = value.find(':', start);
  }
  // zstd_max_train_bytes is optional for backwards compatibility
  if (end != std::string::npos) {
    start = end + 1;
    if (start >= value.size()) {
      return Status::InvalidArgument(
          "unable to parse the specified CF option " + name);
    }
    compression_opts.zstd_max_train_bytes =
        ParseInt(value.substr(start, value.size() - start));
    end = value.find(':', start);
  }
825 826 827 828 829 830 831 832 833 834 835
  // parallel_threads is optional for backwards compatibility
  if (end != std::string::npos) {
    start = end + 1;
    if (start >= value.size()) {
      return Status::InvalidArgument(
          "unable to parse the specified CF option " + name);
    }
    compression_opts.parallel_threads =
        ParseInt(value.substr(start, value.size() - start));
    end = value.find(':', start);
  }
836 837 838 839 840 841 842 843 844 845 846 847 848
  // enabled is optional for backwards compatibility
  if (end != std::string::npos) {
    start = end + 1;
    if (start >= value.size()) {
      return Status::InvalidArgument(
          "unable to parse the specified CF option " + name);
    }
    compression_opts.enabled =
        ParseBoolean("", value.substr(start, value.size() - start));
  }
  return Status::OK();
}

849
Status GetMutableOptionsFromStrings(
850 851
    const MutableCFOptions& base_options,
    const std::unordered_map<std::string, std::string>& options_map,
852
    Logger* info_log, MutableCFOptions* new_options) {
853 854
  assert(new_options);
  *new_options = base_options;
855
  for (const auto& o : options_map) {
856 857 858
    auto& option_name = o.first;
    auto& option_value = o.second;

859
    try {
860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878
      if (option_name == kOptNameBMCompOpts) {
        Status s =
            ParseCompressionOptions(option_value, option_name,
                                    new_options->bottommost_compression_opts);
        if (!s.ok()) {
          return s;
        }
      } else if (option_name == kOptNameCompOpts) {
        Status s = ParseCompressionOptions(option_value, option_name,
                                           new_options->compression_opts);
        if (!s.ok()) {
          return s;
        }
      } else {
        auto iter = cf_options_type_info.find(option_name);
        if (iter == cf_options_type_info.end()) {
          return Status::InvalidArgument("Unrecognized option: " + option_name);
        }
        const auto& opt_info = iter->second;
879
        if (!opt_info.IsMutable()) {
880 881 882
          return Status::InvalidArgument("Option not changeable: " +
                                         option_name);
        }
883
        if (opt_info.IsDeprecated()) {
884 885 886 887 888 889 890 891 892 893 894 895 896
          // log warning when user tries to set a deprecated option but don't
          // fail the call for compatibility.
          ROCKS_LOG_WARN(info_log,
                         "%s is a deprecated option and cannot be set",
                         option_name.c_str());
          continue;
        }
        bool is_ok = ParseOptionHelper(
            reinterpret_cast<char*>(new_options) + opt_info.mutable_offset,
            opt_info.type, option_value);
        if (!is_ok) {
          return Status::InvalidArgument("Error parsing " + option_name);
        }
897
      }
898
    } catch (std::exception& e) {
899
      return Status::InvalidArgument("Error parsing " + option_name + ":" +
900
                                     std::string(e.what()));
901 902
    }
  }
903
  return Status::OK();
904 905
}

906 907 908 909 910 911 912 913 914 915 916 917 918
Status GetMutableDBOptionsFromStrings(
    const MutableDBOptions& base_options,
    const std::unordered_map<std::string, std::string>& options_map,
    MutableDBOptions* new_options) {
  assert(new_options);
  *new_options = base_options;
  for (const auto& o : options_map) {
    try {
      auto iter = db_options_type_info.find(o.first);
      if (iter == db_options_type_info.end()) {
        return Status::InvalidArgument("Unrecognized option: " + o.first);
      }
      const auto& opt_info = iter->second;
919
      if (!opt_info.IsMutable()) {
920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935
        return Status::InvalidArgument("Option not changeable: " + o.first);
      }
      bool is_ok = ParseOptionHelper(
          reinterpret_cast<char*>(new_options) + opt_info.mutable_offset,
          opt_info.type, o.second);
      if (!is_ok) {
        return Status::InvalidArgument("Error parsing " + o.first);
      }
    } catch (std::exception& e) {
      return Status::InvalidArgument("Error parsing " + o.first + ":" +
                                     std::string(e.what()));
    }
  }
  return Status::OK();
}

936 937
Status StringToMap(const std::string& opts_str,
                   std::unordered_map<std::string, std::string>* opts_map) {
L
Lei Jin 已提交
938 939
  assert(opts_map);
  // Example:
940 941
  //   opts_str = "write_buffer_size=1024;max_write_buffer_number=2;"
  //              "nested_opt={opt1=1;opt2=2};max_bytes_for_level_base=100"
L
Lei Jin 已提交
942 943 944 945 946
  size_t pos = 0;
  std::string opts = trim(opts_str);
  while (pos < opts.size()) {
    size_t eq_pos = opts.find('=', pos);
    if (eq_pos == std::string::npos) {
947
      return Status::InvalidArgument("Mismatched key value pair, '=' expected");
L
Lei Jin 已提交
948 949
    }
    std::string key = trim(opts.substr(pos, eq_pos - pos));
950 951 952
    if (key.empty()) {
      return Status::InvalidArgument("Empty key found");
    }
L
Lei Jin 已提交
953

954 955 956 957 958 959 960 961
    // skip space after '=' and look for '{' for possible nested options
    pos = eq_pos + 1;
    while (pos < opts.size() && isspace(opts[pos])) {
      ++pos;
    }
    // Empty value at the end
    if (pos >= opts.size()) {
      (*opts_map)[key] = "";
L
Lei Jin 已提交
962
      break;
963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995
    }
    if (opts[pos] == '{') {
      int count = 1;
      size_t brace_pos = pos + 1;
      while (brace_pos < opts.size()) {
        if (opts[brace_pos] == '{') {
          ++count;
        } else if (opts[brace_pos] == '}') {
          --count;
          if (count == 0) {
            break;
          }
        }
        ++brace_pos;
      }
      // found the matching closing brace
      if (count == 0) {
        (*opts_map)[key] = trim(opts.substr(pos + 1, brace_pos - pos - 1));
        // skip all whitespace and move to the next ';'
        // brace_pos points to the next position after the matching '}'
        pos = brace_pos + 1;
        while (pos < opts.size() && isspace(opts[pos])) {
          ++pos;
        }
        if (pos < opts.size() && opts[pos] != ';') {
          return Status::InvalidArgument(
              "Unexpected chars after nested options");
        }
        ++pos;
      } else {
        return Status::InvalidArgument(
            "Mismatched curly braces for nested options");
      }
L
Lei Jin 已提交
996
    } else {
997 998 999 1000 1001 1002 1003 1004 1005
      size_t sc_pos = opts.find(';', pos);
      if (sc_pos == std::string::npos) {
        (*opts_map)[key] = trim(opts.substr(pos));
        // It either ends with a trailing semi-colon or the last key-value pair
        break;
      } else {
        (*opts_map)[key] = trim(opts.substr(pos, sc_pos - pos));
      }
      pos = sc_pos + 1;
L
Lei Jin 已提交
1006 1007 1008
    }
  }

1009
  return Status::OK();
L
Lei Jin 已提交
1010 1011
}

1012
Status GetStringFromStruct(
1013
    std::string* opt_string, const void* const options,
1014
    const std::unordered_map<std::string, OptionTypeInfo>& type_info,
1015
    const std::string& delimiter) {
1016 1017
  assert(opt_string);
  opt_string->clear();
1018 1019 1020
  for (const auto iter : type_info) {
    const auto& opt_info = iter.second;
    if (opt_info.IsDeprecated()) {
1021 1022 1023 1024
      // If the option is no longer used in rocksdb and marked as deprecated,
      // we skip it in the serialization.
      continue;
    }
1025 1026 1027 1028 1029
    const char* opt_address =
        reinterpret_cast<const char*>(options) + opt_info.offset;
    std::string value;
    bool result =
        SerializeSingleOptionHelper(opt_address, opt_info.type, &value);
1030
    if (result) {
1031
      opt_string->append(iter.first + "=" + value + delimiter);
1032 1033
    } else {
      return Status::InvalidArgument("failed to serialize %s\n",
1034
                                     iter.first.c_str());
1035 1036 1037 1038 1039
    }
  }
  return Status::OK();
}

1040 1041 1042
Status GetStringFromDBOptions(std::string* opt_string,
                              const DBOptions& db_options,
                              const std::string& delimiter) {
1043 1044
  return GetStringFromStruct(opt_string, &db_options, db_options_type_info,
                             delimiter);
1045 1046
}

1047 1048 1049
Status GetStringFromColumnFamilyOptions(std::string* opt_string,
                                        const ColumnFamilyOptions& cf_options,
                                        const std::string& delimiter) {
1050 1051
  return GetStringFromStruct(opt_string, &cf_options, cf_options_type_info,
                             delimiter);
1052 1053
}

1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064
Status GetStringFromCompressionType(std::string* compression_str,
                                    CompressionType compression_type) {
  bool ok = SerializeEnum<CompressionType>(compression_type_string_map,
                                           compression_type, compression_str);
  if (ok) {
    return Status::OK();
  } else {
    return Status::InvalidArgument("Invalid compression types");
  }
}

1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075
std::vector<CompressionType> GetSupportedCompressions() {
  std::vector<CompressionType> supported_compressions;
  for (const auto& comp_to_name : compression_type_string_map) {
    CompressionType t = comp_to_name.second;
    if (t != kDisableCompressionOption && CompressionTypeSupported(t)) {
      supported_compressions.push_back(t);
    }
  }
  return supported_compressions;
}

1076 1077 1078 1079
Status ParseDBOption(const std::string& name,
                     const std::string& org_value,
                     DBOptions* new_options,
                     bool input_strings_escaped = false) {
1080
  const std::string& value =
1081
      input_strings_escaped ? UnescapeOptionString(org_value) : org_value;
1082
  try {
1083
    if (name == "rate_limiter_bytes_per_sec") {
I
Igor Canadi 已提交
1084 1085
      new_options->rate_limiter.reset(
          NewGenericRateLimiter(static_cast<int64_t>(ParseUint64(value))));
1086 1087
    } else if (name == kNameEnv) {
      // Currently `Env` can be deserialized from object registry only.
1088 1089
      Env* env = new_options->env;
      Status status = Env::LoadEnv(value, &env);
1090
      // Only support static env for now.
1091
      if (status.ok()) {
1092 1093
        new_options->env = env;
      }
1094
    } else {
1095 1096
      auto iter = db_options_type_info.find(name);
      if (iter == db_options_type_info.end()) {
1097
        return Status::InvalidArgument("Unrecognized option DBOptions:", name);
1098
      }
1099
      const auto& opt_info = iter->second;
1100
      if (opt_info.IsDeprecated() ||
1101
          ParseOptionHelper(
1102 1103 1104
              reinterpret_cast<char*>(new_options) + opt_info.offset,
              opt_info.type, value)) {
        return Status::OK();
1105 1106 1107 1108 1109 1110
      } else if (opt_info.IsByName()) {
        return Status::NotSupported("Deserializing the specified DB option " +
                                    name + " is not supported");
      } else {
        return Status::InvalidArgument(
            "Unable to parse the specified DB option " + name);
1111
      }
1112
    }
D
Dmitri Smirnov 已提交
1113
  } catch (const std::exception&) {
1114
    return Status::InvalidArgument("Unable to parse DBOptions:", name);
1115
  }
1116
  return Status::OK();
1117
}
L
Lei Jin 已提交
1118

1119
Status GetColumnFamilyOptionsFromMap(
L
Lei Jin 已提交
1120 1121
    const ColumnFamilyOptions& base_options,
    const std::unordered_map<std::string, std::string>& opts_map,
1122 1123
    ColumnFamilyOptions* new_options, bool input_strings_escaped,
    bool ignore_unknown_options) {
1124
  return GetColumnFamilyOptionsFromMapInternal(
1125 1126
      base_options, opts_map, new_options, input_strings_escaped, nullptr,
      ignore_unknown_options);
1127 1128 1129 1130 1131 1132
}

Status GetColumnFamilyOptionsFromMapInternal(
    const ColumnFamilyOptions& base_options,
    const std::unordered_map<std::string, std::string>& opts_map,
    ColumnFamilyOptions* new_options, bool input_strings_escaped,
1133 1134
    std::vector<std::string>* unsupported_options_names,
    bool ignore_unknown_options) {
1135 1136
  assert(new_options);
  *new_options = base_options;
1137 1138 1139
  if (unsupported_options_names) {
    unsupported_options_names->clear();
  }
L
Lei Jin 已提交
1140
  for (const auto& o : opts_map) {
1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153
    auto s = ParseColumnFamilyOption(o.first, o.second, new_options,
                                 input_strings_escaped);
    if (!s.ok()) {
      if (s.IsNotSupported()) {
        // If the deserialization of the specified option is not supported
        // and an output vector of unsupported_options is provided, then
        // we log the name of the unsupported option and proceed.
        if (unsupported_options_names != nullptr) {
          unsupported_options_names->push_back(o.first);
        }
        // Note that we still return Status::OK in such case to maintain
        // the backward compatibility in the old public API defined in
        // rocksdb/convenience.h
1154 1155
      } else if (s.IsInvalidArgument() && ignore_unknown_options) {
        continue;
1156
      } else {
1157 1158
        // Restore "new_options" to the default "base_options".
        *new_options = base_options;
1159
        return s;
1160
      }
L
Lei Jin 已提交
1161 1162
    }
  }
1163
  return Status::OK();
L
Lei Jin 已提交
1164 1165
}

1166
Status GetColumnFamilyOptionsFromString(
L
Lei Jin 已提交
1167 1168 1169 1170
    const ColumnFamilyOptions& base_options,
    const std::string& opts_str,
    ColumnFamilyOptions* new_options) {
  std::unordered_map<std::string, std::string> opts_map;
1171 1172
  Status s = StringToMap(opts_str, &opts_map);
  if (!s.ok()) {
1173
    *new_options = base_options;
1174
    return s;
L
Lei Jin 已提交
1175 1176 1177 1178
  }
  return GetColumnFamilyOptionsFromMap(base_options, opts_map, new_options);
}

1179
Status GetDBOptionsFromMap(
L
Lei Jin 已提交
1180 1181
    const DBOptions& base_options,
    const std::unordered_map<std::string, std::string>& opts_map,
1182 1183 1184 1185 1186
    DBOptions* new_options, bool input_strings_escaped,
    bool ignore_unknown_options) {
  return GetDBOptionsFromMapInternal(base_options, opts_map, new_options,
                                     input_strings_escaped, nullptr,
                                     ignore_unknown_options);
1187 1188 1189 1190 1191 1192
}

Status GetDBOptionsFromMapInternal(
    const DBOptions& base_options,
    const std::unordered_map<std::string, std::string>& opts_map,
    DBOptions* new_options, bool input_strings_escaped,
1193 1194
    std::vector<std::string>* unsupported_options_names,
    bool ignore_unknown_options) {
L
Lei Jin 已提交
1195 1196
  assert(new_options);
  *new_options = base_options;
1197 1198 1199
  if (unsupported_options_names) {
    unsupported_options_names->clear();
  }
L
Lei Jin 已提交
1200
  for (const auto& o : opts_map) {
1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213
    auto s = ParseDBOption(o.first, o.second,
                           new_options, input_strings_escaped);
    if (!s.ok()) {
      if (s.IsNotSupported()) {
        // If the deserialization of the specified option is not supported
        // and an output vector of unsupported_options is provided, then
        // we log the name of the unsupported option and proceed.
        if (unsupported_options_names != nullptr) {
          unsupported_options_names->push_back(o.first);
        }
        // Note that we still return Status::OK in such case to maintain
        // the backward compatibility in the old public API defined in
        // rocksdb/convenience.h
1214 1215
      } else if (s.IsInvalidArgument() && ignore_unknown_options) {
        continue;
1216
      } else {
1217 1218
        // Restore "new_options" to the default "base_options".
        *new_options = base_options;
1219 1220
        return s;
      }
1221 1222
    }
  }
1223
  return Status::OK();
1224 1225
}

1226
Status GetDBOptionsFromString(
L
Lei Jin 已提交
1227 1228 1229 1230
    const DBOptions& base_options,
    const std::string& opts_str,
    DBOptions* new_options) {
  std::unordered_map<std::string, std::string> opts_map;
1231 1232
  Status s = StringToMap(opts_str, &opts_map);
  if (!s.ok()) {
1233
    *new_options = base_options;
1234
    return s;
L
Lei Jin 已提交
1235 1236 1237 1238
  }
  return GetDBOptionsFromMap(base_options, opts_map, new_options);
}

1239 1240 1241 1242 1243 1244 1245 1246 1247 1248
Status GetOptionsFromString(const Options& base_options,
                            const std::string& opts_str, Options* new_options) {
  std::unordered_map<std::string, std::string> opts_map;
  Status s = StringToMap(opts_str, &opts_map);
  if (!s.ok()) {
    return s;
  }
  DBOptions new_db_options(base_options);
  ColumnFamilyOptions new_cf_options(base_options);
  for (const auto& o : opts_map) {
1249 1250 1251
    if (ParseDBOption(o.first, o.second, &new_db_options).ok()) {
    } else if (ParseColumnFamilyOption(
        o.first, o.second, &new_cf_options).ok()) {
1252 1253 1254 1255 1256 1257 1258 1259
    } else {
      return Status::InvalidArgument("Can't parse option " + o.first);
    }
  }
  *new_options = Options(new_db_options, new_cf_options);
  return Status::OK();
}

1260 1261 1262
Status GetTableFactoryFromMap(
    const std::string& factory_name,
    const std::unordered_map<std::string, std::string>& opt_map,
1263
    std::shared_ptr<TableFactory>* table_factory, bool ignore_unknown_options) {
1264 1265 1266 1267
  Status s;
  if (factory_name == BlockBasedTableFactory().Name()) {
    BlockBasedTableOptions bbt_opt;
    s = GetBlockBasedTableOptionsFromMap(BlockBasedTableOptions(), opt_map,
1268 1269 1270
                                         &bbt_opt,
                                         true, /* input_strings_escaped */
                                         ignore_unknown_options);
1271 1272 1273 1274 1275
    if (!s.ok()) {
      return s;
    }
    table_factory->reset(new BlockBasedTableFactory(bbt_opt));
    return Status::OK();
1276 1277
  } else if (factory_name == PlainTableFactory().Name()) {
    PlainTableOptions pt_opt;
I
Islam AbdelRahman 已提交
1278
    s = GetPlainTableOptionsFromMap(PlainTableOptions(), opt_map, &pt_opt,
1279 1280
                                    true, /* input_strings_escaped */
                                    ignore_unknown_options);
1281 1282 1283 1284 1285
    if (!s.ok()) {
      return s;
    }
    table_factory->reset(new PlainTableFactory(pt_opt));
    return Status::OK();
1286 1287 1288 1289 1290 1291 1292
  }
  // Return OK for not supported table factories as TableFactory
  // Deserialization is optional.
  table_factory->reset();
  return Status::OK();
}

1293 1294 1295 1296 1297
std::unordered_map<std::string, BlockBasedTableOptions::IndexType>
    OptionsHelper::block_base_table_index_type_string_map = {
        {"kBinarySearch", BlockBasedTableOptions::IndexType::kBinarySearch},
        {"kHashSearch", BlockBasedTableOptions::IndexType::kHashSearch},
        {"kTwoLevelIndexSearch",
1298 1299 1300
         BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch},
        {"kBinarySearchWithFirstKey",
         BlockBasedTableOptions::IndexType::kBinarySearchWithFirstKey}};
1301

1302 1303 1304 1305
std::unordered_map<std::string, BlockBasedTableOptions::DataBlockIndexType>
    OptionsHelper::block_base_table_data_block_index_type_string_map = {
        {"kDataBlockBinarySearch",
         BlockBasedTableOptions::DataBlockIndexType::kDataBlockBinarySearch},
1306 1307
        {"kDataBlockBinaryAndHash",
         BlockBasedTableOptions::DataBlockIndexType::kDataBlockBinaryAndHash}};
1308

1309 1310
std::unordered_map<std::string, BlockBasedTableOptions::IndexShorteningMode>
    OptionsHelper::block_base_table_index_shortening_mode_string_map = {
1311 1312 1313 1314 1315 1316 1317
        {"kNoShortening",
         BlockBasedTableOptions::IndexShorteningMode::kNoShortening},
        {"kShortenSeparators",
         BlockBasedTableOptions::IndexShorteningMode::kShortenSeparators},
        {"kShortenSeparatorsAndSuccessor",
         BlockBasedTableOptions::IndexShorteningMode::
             kShortenSeparatorsAndSuccessor}};
1318

1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361
std::unordered_map<std::string, EncodingType>
    OptionsHelper::encoding_type_string_map = {{"kPlain", kPlain},
                                               {"kPrefix", kPrefix}};

std::unordered_map<std::string, CompactionStyle>
    OptionsHelper::compaction_style_string_map = {
        {"kCompactionStyleLevel", kCompactionStyleLevel},
        {"kCompactionStyleUniversal", kCompactionStyleUniversal},
        {"kCompactionStyleFIFO", kCompactionStyleFIFO},
        {"kCompactionStyleNone", kCompactionStyleNone}};

std::unordered_map<std::string, CompactionPri>
    OptionsHelper::compaction_pri_string_map = {
        {"kByCompensatedSize", kByCompensatedSize},
        {"kOldestLargestSeqFirst", kOldestLargestSeqFirst},
        {"kOldestSmallestSeqFirst", kOldestSmallestSeqFirst},
        {"kMinOverlappingRatio", kMinOverlappingRatio}};

std::unordered_map<std::string, WALRecoveryMode>
    OptionsHelper::wal_recovery_mode_string_map = {
        {"kTolerateCorruptedTailRecords",
         WALRecoveryMode::kTolerateCorruptedTailRecords},
        {"kAbsoluteConsistency", WALRecoveryMode::kAbsoluteConsistency},
        {"kPointInTimeRecovery", WALRecoveryMode::kPointInTimeRecovery},
        {"kSkipAnyCorruptedRecords",
         WALRecoveryMode::kSkipAnyCorruptedRecords}};

std::unordered_map<std::string, DBOptions::AccessHint>
    OptionsHelper::access_hint_string_map = {
        {"NONE", DBOptions::AccessHint::NONE},
        {"NORMAL", DBOptions::AccessHint::NORMAL},
        {"SEQUENTIAL", DBOptions::AccessHint::SEQUENTIAL},
        {"WILLNEED", DBOptions::AccessHint::WILLNEED}};

std::unordered_map<std::string, InfoLogLevel>
    OptionsHelper::info_log_level_string_map = {
        {"DEBUG_LEVEL", InfoLogLevel::DEBUG_LEVEL},
        {"INFO_LEVEL", InfoLogLevel::INFO_LEVEL},
        {"WARN_LEVEL", InfoLogLevel::WARN_LEVEL},
        {"ERROR_LEVEL", InfoLogLevel::ERROR_LEVEL},
        {"FATAL_LEVEL", InfoLogLevel::FATAL_LEVEL},
        {"HEADER_LEVEL", InfoLogLevel::HEADER_LEVEL}};

1362
LRUCacheOptions OptionsHelper::dummy_lru_cache_options;
1363
CompactionOptionsUniversal OptionsHelper::dummy_comp_options_universal;
1364
CompactionOptionsFIFO OptionsHelper::dummy_comp_options;
1365 1366

template <typename T1>
1367 1368 1369
int offset_of(T1 LRUCacheOptions::*member) {
  return int(size_t(&(OptionsHelper::dummy_lru_cache_options.*member)) -
             size_t(&OptionsHelper::dummy_lru_cache_options));
1370
}
1371

1372 1373 1374 1375 1376
template <typename T1>
int offset_of(T1 CompactionOptionsFIFO::*member) {
  return int(size_t(&(OptionsHelper::dummy_comp_options.*member)) -
             size_t(&OptionsHelper::dummy_comp_options));
}
1377
template <typename T1>
1378 1379 1380 1381
int offset_of(T1 CompactionOptionsUniversal::*member) {
  return int(size_t(&(OptionsHelper::dummy_comp_options_universal.*member)) -
             size_t(&OptionsHelper::dummy_comp_options_universal));
}
1382 1383 1384 1385 1386

std::unordered_map<std::string, OptionTypeInfo>
    OptionsHelper::fifo_compaction_options_type_info = {
        {"max_table_files_size",
         {offset_of(&CompactionOptionsFIFO::max_table_files_size),
1387 1388
          OptionType::kUInt64T, OptionVerificationType::kNormal,
          OptionTypeFlags::kMutable,
1389 1390
          offsetof(struct CompactionOptionsFIFO, max_table_files_size)}},
        {"ttl",
1391 1392
         {0, OptionType::kUInt64T, OptionVerificationType::kDeprecated,
          OptionTypeFlags::kNone, 0}},
1393 1394
        {"allow_compaction",
         {offset_of(&CompactionOptionsFIFO::allow_compaction),
1395 1396
          OptionType::kBoolean, OptionVerificationType::kNormal,
          OptionTypeFlags::kMutable,
1397 1398
          offsetof(struct CompactionOptionsFIFO, allow_compaction)}}};

1399 1400 1401 1402
std::unordered_map<std::string, OptionTypeInfo>
    OptionsHelper::universal_compaction_options_type_info = {
        {"size_ratio",
         {offset_of(&CompactionOptionsUniversal::size_ratio), OptionType::kUInt,
1403
          OptionVerificationType::kNormal, OptionTypeFlags::kMutable,
1404 1405 1406
          offsetof(class CompactionOptionsUniversal, size_ratio)}},
        {"min_merge_width",
         {offset_of(&CompactionOptionsUniversal::min_merge_width),
1407 1408
          OptionType::kUInt, OptionVerificationType::kNormal,
          OptionTypeFlags::kMutable,
1409 1410 1411
          offsetof(class CompactionOptionsUniversal, min_merge_width)}},
        {"max_merge_width",
         {offset_of(&CompactionOptionsUniversal::max_merge_width),
1412 1413
          OptionType::kUInt, OptionVerificationType::kNormal,
          OptionTypeFlags::kMutable,
1414 1415 1416 1417
          offsetof(class CompactionOptionsUniversal, max_merge_width)}},
        {"max_size_amplification_percent",
         {offset_of(
              &CompactionOptionsUniversal::max_size_amplification_percent),
1418 1419
          OptionType::kUInt, OptionVerificationType::kNormal,
          OptionTypeFlags::kMutable,
1420 1421 1422 1423
          offsetof(class CompactionOptionsUniversal,
                   max_size_amplification_percent)}},
        {"compression_size_percent",
         {offset_of(&CompactionOptionsUniversal::compression_size_percent),
1424 1425
          OptionType::kInt, OptionVerificationType::kNormal,
          OptionTypeFlags::kMutable,
1426 1427 1428 1429 1430
          offsetof(class CompactionOptionsUniversal,
                   compression_size_percent)}},
        {"stop_style",
         {offset_of(&CompactionOptionsUniversal::stop_style),
          OptionType::kCompactionStopStyle, OptionVerificationType::kNormal,
1431 1432
          OptionTypeFlags::kMutable,
          offsetof(class CompactionOptionsUniversal, stop_style)}},
1433 1434
        {"allow_trivial_move",
         {offset_of(&CompactionOptionsUniversal::allow_trivial_move),
1435 1436
          OptionType::kBoolean, OptionVerificationType::kNormal,
          OptionTypeFlags::kMutable,
1437 1438 1439 1440 1441 1442 1443
          offsetof(class CompactionOptionsUniversal, allow_trivial_move)}}};

std::unordered_map<std::string, CompactionStopStyle>
    OptionsHelper::compaction_stop_style_string_map = {
        {"kCompactionStopStyleSimilarSize", kCompactionStopStyleSimilarSize},
        {"kCompactionStopStyleTotalSize", kCompactionStopStyleTotalSize}};

1444 1445
std::unordered_map<std::string, OptionTypeInfo>
    OptionsHelper::lru_cache_options_type_info = {
1446 1447
        {"capacity",
         {offset_of(&LRUCacheOptions::capacity), OptionType::kSizeT,
1448
          OptionVerificationType::kNormal, OptionTypeFlags::kMutable,
1449
          offsetof(struct LRUCacheOptions, capacity)}},
1450 1451
        {"num_shard_bits",
         {offset_of(&LRUCacheOptions::num_shard_bits), OptionType::kInt,
1452
          OptionVerificationType::kNormal, OptionTypeFlags::kMutable,
1453 1454 1455
          offsetof(struct LRUCacheOptions, num_shard_bits)}},
        {"strict_capacity_limit",
         {offset_of(&LRUCacheOptions::strict_capacity_limit),
1456 1457
          OptionType::kBoolean, OptionVerificationType::kNormal,
          OptionTypeFlags::kMutable,
1458 1459
          offsetof(struct LRUCacheOptions, strict_capacity_limit)}},
        {"high_pri_pool_ratio",
1460
         {offset_of(&LRUCacheOptions::high_pri_pool_ratio), OptionType::kDouble,
1461
          OptionVerificationType::kNormal, OptionTypeFlags::kMutable,
1462 1463
          offsetof(struct LRUCacheOptions, high_pri_pool_ratio)}}};

1464
#endif  // !ROCKSDB_LITE
1465

1466
}  // namespace ROCKSDB_NAMESPACE