From e114f0abb850c31f794337bf249574ccc8cee7ca Mon Sep 17 00:00:00 2001 From: Yueh-Hsuan Chiang Date: Tue, 10 Nov 2015 22:58:01 -0800 Subject: [PATCH] Enable RocksDB to persist Options file. Summary: This patch allows rocksdb to persist options into a file on DB::Open, SetOptions, and Create / Drop ColumnFamily. Options files are created under the same directory as the rocksdb instance. In addition, this patch also adds a fail_if_missing_options_file in DBOptions that makes any function call return non-ok status when it is not able to persist options properly. // If true, then DB::Open / CreateColumnFamily / DropColumnFamily // / SetOptions will fail if options file is not detected or properly // persisted. // // DEFAULT: false bool fail_if_missing_options_file; Options file names are formatted as OPTIONS-, and RocksDB will always keep the latest two options files. Test Plan: Add options_file_test. options_test column_family_test Reviewers: igor, IslamAbdelRahman, sdong, anthony Reviewed By: anthony Subscribers: dhruba Differential Revision: https://reviews.facebook.net/D48285 --- CMakeLists.txt | 1 + HISTORY.md | 1 + Makefile | 4 + db/column_family_test.cc | 1 + db/db_impl.cc | 170 ++++++++++++++++++++- db/db_impl.h | 12 ++ db/db_test_util.cc | 1 + db/filename.cc | 40 ++++- db/filename.h | 16 +- db/options_file_test.cc | 116 ++++++++++++++ include/rocksdb/options.h | 7 + include/rocksdb/slice.h | 10 ++ include/rocksdb/status.h | 2 +- src.mk | 1 + util/options.cc | 14 +- util/options_helper.cc | 66 ++++++++ util/options_helper.h | 3 + utilities/backupable/backupable_db_test.cc | 8 +- 18 files changed, 455 insertions(+), 18 deletions(-) create mode 100644 db/options_file_test.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 808f2a64a..ad99eba36 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -329,6 +329,7 @@ set(TESTS db/memtable_list_test.cc db/merge_test.cc db/merge_helper_test.cc + db/options_file_test.cc db/perf_context_test.cc db/plain_table_db_test.cc db/prefix_test.cc diff --git a/HISTORY.md b/HISTORY.md index f4227078c..a54c13da4 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -5,6 +5,7 @@ * Introduce CreateLoggerFromOptions(), this function create a Logger for provided DBOptions. * Add GetAggregatedIntProperty(), which returns the sum of the GetIntProperty of all the column families. * Add MemoryUtil in rocksdb/utilities/memory.h. It currently offers a way to get the memory usage by type from a list rocksdb instances. +* RocksDB will now persist options under the same directory as the RocksDB database on successful DB::Open, CreateColumnFamily, DropColumnFamily, and SetOptions. ### Public API Changes * CompactionFilter::Context includes information of Column Family ID * The need-compaction hint given by TablePropertiesCollector::NeedCompact() will be persistent and recoverable after DB recovery. This introduces a breaking format change. If you use this experimental feature, including NewCompactOnDeletionCollectorFactory() in the new version, you may not be able to directly downgrade the DB back to version 4.0 or lower. diff --git a/Makefile b/Makefile index bbb59891f..aae039663 100644 --- a/Makefile +++ b/Makefile @@ -275,6 +275,7 @@ TESTS = \ memory_test \ merge_test \ merger_test \ + options_file_test \ redis_test \ reduce_levels_test \ plain_table_db_test \ @@ -892,6 +893,9 @@ merge_test: db/merge_test.o $(LIBOBJECTS) $(TESTHARNESS) merger_test: table/merger_test.o $(LIBOBJECTS) $(TESTHARNESS) $(AM_LINK) +options_file_test: db/options_file_test.o $(LIBOBJECTS) $(TESTHARNESS) + $(AM_LINK) + deletefile_test: db/deletefile_test.o $(LIBOBJECTS) $(TESTHARNESS) $(AM_LINK) diff --git a/db/column_family_test.cc b/db/column_family_test.cc index 938c4121a..e776ae115 100644 --- a/db/column_family_test.cc +++ b/db/column_family_test.cc @@ -57,6 +57,7 @@ class ColumnFamilyTest : public testing::Test { env_ = new EnvCounter(Env::Default()); dbname_ = test::TmpDir() + "/column_family_test"; db_options_.create_if_missing = true; + db_options_.fail_if_options_file_error = true; db_options_.env = env_; DestroyDB(dbname_, Options(db_options_, column_family_options_)); } diff --git a/db/db_impl.cc b/db/db_impl.cc index 40b7ae817..a250dbbcb 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -84,6 +85,8 @@ #include "util/log_buffer.h" #include "util/logging.h" #include "util/mutexlock.h" +#include "util/options_helper.h" +#include "util/options_parser.h" #include "util/perf_context_imp.h" #include "util/stop_watch.h" #include "util/string_util.h" @@ -734,8 +737,12 @@ void DBImpl::PurgeObsoleteFiles(const JobContext& state) { // Also, SetCurrentFile creates a temp file when writing out new // manifest, which is equal to state.pending_manifest_file_number. We // should not delete that file + // + // TODO(yhchiang): carefully modify the third condition to safely + // remove the temp options files. keep = (sst_live_map.find(number) != sst_live_map.end()) || - (number == state.pending_manifest_file_number); + (number == state.pending_manifest_file_number) || + (to_delete.find(kOptionsFileNamePrefix) != std::string::npos); break; case kInfoLogFile: keep = true; @@ -747,6 +754,7 @@ void DBImpl::PurgeObsoleteFiles(const JobContext& state) { case kDBLockFile: case kIdentityFile: case kMetaDatabase: + case kOptionsFile: keep = true; break; } @@ -1922,6 +1930,19 @@ Status DBImpl::SetOptions(ColumnFamilyHandle* column_family, new_options = *cfd->GetLatestMutableCFOptions(); } } + if (s.ok()) { + Status persist_options_status = WriteOptionsFile(); + if (!persist_options_status.ok()) { + if (db_options_.fail_if_options_file_error) { + s = Status::IOError( + "SetOptions succeeded, but unable to persist options", + persist_options_status.ToString()); + } + Warn(db_options_.info_log, + "Unable to persist options in SetOptions() -- %s", + persist_options_status.ToString().c_str()); + } + } Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log, "SetOptions() on column family [%s], inputs:", @@ -3458,6 +3479,18 @@ Status DBImpl::CreateColumnFamily(const ColumnFamilyOptions& cf_options, // this is outside the mutex if (s.ok()) { + Status persist_options_status = WriteOptionsFile(); + if (!persist_options_status.ok()) { + if (db_options_.fail_if_options_file_error) { + s = Status::IOError( + "ColumnFamily has been created, but unable to persist" + "options in CreateColumnFamily()", + persist_options_status.ToString().c_str()); + } + Warn(db_options_.info_log, + "Unable to persist options in CreateColumnFamily() -- %s", + persist_options_status.ToString().c_str()); + } NewThreadStatusCfInfo( reinterpret_cast(*handle)->cfd()); } @@ -3515,6 +3548,18 @@ Status DBImpl::DropColumnFamily(ColumnFamilyHandle* column_family) { auto* mutable_cf_options = cfd->GetLatestMutableCFOptions(); max_total_in_memory_state_ -= mutable_cf_options->write_buffer_size * mutable_cf_options->max_write_buffer_number; + auto options_persist_status = WriteOptionsFile(); + if (!options_persist_status.ok()) { + if (db_options_.fail_if_options_file_error) { + s = Status::IOError( + "ColumnFamily has been dropped, but unable to persist " + "options in DropColumnFamily()", + options_persist_status.ToString().c_str()); + } + Warn(db_options_.info_log, + "Unable to persist options in DropColumnFamily() -- %s", + options_persist_status.ToString().c_str()); + } Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log, "Dropped column family with id %u\n", cfd->GetID()); @@ -4931,6 +4976,19 @@ Status DB::Open(const DBOptions& db_options, const std::string& dbname, impl); LogFlush(impl->db_options_.info_log); + auto persist_options_status = impl->WriteOptionsFile(); + if (!persist_options_status.ok()) { + if (db_options.fail_if_options_file_error) { + s = Status::IOError( + "DB::Open() failed --- Unable to persist Options file", + persist_options_status.ToString()); + } + Warn(impl->db_options_.info_log, + "Unable to persist options in DB::Open() -- %s", + persist_options_status.ToString().c_str()); + } + } + if (s.ok()) { *dbptr = impl; } else { for (auto* h : *handles) { @@ -4938,6 +4996,7 @@ Status DB::Open(const DBOptions& db_options, const std::string& dbname, } handles->clear(); delete impl; + *dbptr = nullptr; } return s; } @@ -5034,6 +5093,7 @@ Status DestroyDB(const std::string& dbname, const Options& options) { } } } + // ignore case where no archival directory is present. env->DeleteDir(archivedir); @@ -5045,6 +5105,114 @@ Status DestroyDB(const std::string& dbname, const Options& options) { return result; } +Status DBImpl::WriteOptionsFile() { +#ifndef ROCKSDB_LITE + std::string file_name; + Status s = WriteOptionsToTempFile(&file_name); + if (!s.ok()) { + return s; + } + s = RenameTempFileToOptionsFile(file_name); + return s; +#else + return Status::OK(); +#endif // !ROCKSDB_LITE +} + +Status DBImpl::WriteOptionsToTempFile(std::string* file_name) { +#ifndef ROCKSDB_LITE + std::vector cf_names; + std::vector cf_opts; + { + InstrumentedMutexLock l(&mutex_); + // This part requires mutex to protect the column family options + for (auto cfd : *versions_->GetColumnFamilySet()) { + if (cfd->IsDropped()) { + continue; + } + cf_names.push_back(cfd->GetName()); + cf_opts.push_back(BuildColumnFamilyOptions( + *cfd->options(), *cfd->GetLatestMutableCFOptions())); + } + } + *file_name = TempOptionsFileName(GetName(), versions_->NewFileNumber()); + + Status s = PersistRocksDBOptions(GetDBOptions(), cf_names, cf_opts, + *file_name, GetEnv()); + return s; +#else + return Status::OK(); +#endif // !ROCKSDB_LITE +} + +#ifndef ROCKSDB_LITE +namespace { +void DeleteOptionsFilesHelper(const std::map& filenames, + const size_t num_files_to_keep, + const std::shared_ptr& info_log, + Env* env) { + if (filenames.size() <= num_files_to_keep) { + return; + } + for (auto iter = std::next(filenames.begin(), num_files_to_keep); + iter != filenames.end(); ++iter) { + if (!env->DeleteFile(iter->second).ok()) { + Warn(info_log, "Unable to delete options file %s", iter->second.c_str()); + } + } +} +} // namespace +#endif // !ROCKSDB_LITE + +Status DBImpl::DeleteObsoleteOptionsFiles() { +#ifndef ROCKSDB_LITE + options_files_mutex_.AssertHeld(); + + std::vector filenames; + // use ordered map to store keep the filenames sorted from the newest + // to the oldest. + std::map options_filenames; + Status s; + s = GetEnv()->GetChildren(GetName(), &filenames); + if (!s.ok()) { + return s; + } + for (auto& filename : filenames) { + uint64_t file_number; + FileType type; + if (ParseFileName(filename, &file_number, &type) && type == kOptionsFile) { + options_filenames.insert( + {std::numeric_limits::max() - file_number, + GetName() + "/" + filename}); + } + } + + // Keeps the latest 2 Options file + const size_t kNumOptionsFilesKept = 2; + DeleteOptionsFilesHelper(options_filenames, kNumOptionsFilesKept, + db_options_.info_log, GetEnv()); + return Status::OK(); +#else + return Status::OK(); +#endif // !ROCKSDB_LITE +} + +Status DBImpl::RenameTempFileToOptionsFile(const std::string& file_name) { +#ifndef ROCKSDB_LITE + InstrumentedMutexLock l(&options_files_mutex_); + Status s; + std::string options_file_name = + OptionsFileName(GetName(), versions_->NewFileNumber()); + // Retry if the file name happen to conflict with an existing one. + s = GetEnv()->RenameFile(file_name, options_file_name); + + DeleteObsoleteOptionsFiles(); + return s; +#else + return Status::OK(); +#endif // !ROCKSDB_LITE +} + #if ROCKSDB_USING_THREAD_STATUS void DBImpl::NewThreadStatusCfInfo( diff --git a/db/db_impl.h b/db/db_impl.h index 3c538dac1..b59cadfc1 100644 --- a/db/db_impl.h +++ b/db/db_impl.h @@ -398,6 +398,13 @@ class DBImpl : public DB { SuperVersion* super_version, Arena* arena); + // The following options file related functions should not be + // called while DB mutex is held. + Status WriteOptionsFile(); + Status WriteOptionsToTempFile(std::string* file_name); + Status RenameTempFileToOptionsFile(const std::string& file_name); + Status DeleteObsoleteOptionsFiles(); + void NotifyOnFlushCompleted(ColumnFamilyData* cfd, FileMetaData* file_meta, const MutableCFOptions& mutable_cf_options, int job_id, TableProperties prop); @@ -552,8 +559,13 @@ class DBImpl : public DB { // Lock over the persistent DB state. Non-nullptr iff successfully acquired. FileLock* db_lock_; + // The mutex for options file related operations. + // NOTE: should never acquire options_file_mutex_ and mutex_ at the + // same time. + InstrumentedMutex options_files_mutex_; // State below is protected by mutex_ InstrumentedMutex mutex_; + std::atomic shutting_down_; // This condition variable is signaled on these conditions: // * whenever bg_compaction_scheduled_ goes down to 0 diff --git a/db/db_test_util.cc b/db/db_test_util.cc index 356398871..6cfb1781a 100644 --- a/db/db_test_util.cc +++ b/db/db_test_util.cc @@ -368,6 +368,7 @@ Options DBTestBase::CurrentOptions( } options.env = env_; options.create_if_missing = true; + options.fail_if_options_file_error = true; return options; } diff --git a/db/filename.cc b/db/filename.cc index f57b178b9..32cd8758a 100644 --- a/db/filename.cc +++ b/db/filename.cc @@ -21,6 +21,7 @@ #include "util/file_reader_writer.h" #include "util/logging.h" #include "util/stop_watch.h" +#include "util/string_util.h" #include "util/sync_point.h" namespace rocksdb { @@ -47,8 +48,9 @@ static size_t GetInfoLogPrefix(const std::string& path, char* dest, int len) { path[i] == '_'){ dest[write_idx++] = path[i]; } else { - if (i > 0) + if (i > 0) { dest[write_idx++] = '_'; + } } i++; } @@ -146,7 +148,7 @@ std::string LockFileName(const std::string& dbname) { } std::string TempFileName(const std::string& dbname, uint64_t number) { - return MakeFileName(dbname, number, "dbtmp"); + return MakeFileName(dbname, number, kTempFileNameSuffix.c_str()); } InfoLogPrefix::InfoLogPrefix(bool has_log_dir, @@ -186,6 +188,21 @@ std::string OldInfoLogFileName(const std::string& dbname, uint64_t ts, return log_dir + "/" + info_log_prefix.buf + ".old." + buf; } +std::string OptionsFileName(const std::string& dbname, uint64_t file_num) { + char buffer[256]; + snprintf(buffer, sizeof(buffer), "%s%06" PRIu64, + kOptionsFileNamePrefix.c_str(), file_num); + return dbname + "/" + buffer; +} + +std::string TempOptionsFileName(const std::string& dbname, uint64_t file_num) { + char buffer[256]; + snprintf(buffer, sizeof(buffer), "%s%06" PRIu64 ".%s", + kOptionsFileNamePrefix.c_str(), file_num, + kTempFileNameSuffix.c_str()); + return dbname + "/" + buffer; +} + std::string MetaDatabaseName(const std::string& dbname, uint64_t number) { char buf[100]; snprintf(buf, sizeof(buf), "/METADB-%llu", @@ -206,6 +223,8 @@ std::string IdentityFileName(const std::string& dbname) { // dbname/MANIFEST-[0-9]+ // dbname/[0-9]+.(log|sst) // dbname/METADB-[0-9]+ +// dbname/OPTIONS-[0-9]+ +// dbname/OPTIONS-[0-9]+.dbtmp // Disregards / at the beginning bool ParseFileName(const std::string& fname, uint64_t* number, @@ -268,6 +287,21 @@ bool ParseFileName(const std::string& fname, uint64_t* number, } *type = kMetaDatabase; *number = num; + } else if (rest.starts_with(kOptionsFileNamePrefix)) { + uint64_t ts_suffix; + bool is_temp_file = false; + rest.remove_prefix(kOptionsFileNamePrefix.size()); + const std::string kTempFileNameSuffixWithDot = + std::string(".") + kTempFileNameSuffix; + if (rest.ends_with(kTempFileNameSuffixWithDot)) { + rest.remove_suffix(kTempFileNameSuffixWithDot.size()); + is_temp_file = true; + } + if (!ConsumeDecimalNumber(&rest, &ts_suffix)) { + return false; + } + *number = ts_suffix; + *type = is_temp_file ? kTempFile : kOptionsFile; } else { // Avoid strtoull() to keep filename format independent of the // current locale @@ -302,7 +336,7 @@ bool ParseFileName(const std::string& fname, uint64_t* number, } else if (suffix == Slice(kRocksDbTFileExt) || suffix == Slice(kLevelDbTFileExt)) { *type = kTableFile; - } else if (suffix == Slice("dbtmp")) { + } else if (suffix == Slice(kTempFileNameSuffix)) { *type = kTempFile; } else { return false; diff --git a/db/filename.h b/db/filename.h index 926f027de..f7196c9f2 100644 --- a/db/filename.h +++ b/db/filename.h @@ -36,7 +36,8 @@ enum FileType { kTempFile, kInfoLogFile, // Either the current one, or an old one kMetaDatabase, - kIdentityFile + kIdentityFile, + kOptionsFile }; // Return the name of the log file with the specified number @@ -114,6 +115,19 @@ extern std::string OldInfoLogFileName(const std::string& dbname, uint64_t ts, const std::string& db_path = "", const std::string& log_dir = ""); +static const std::string kOptionsFileNamePrefix = "OPTIONS-"; +static const std::string kTempFileNameSuffix = "dbtmp"; + +// Return a options file name given the "dbname" and file number. +// Format: OPTIONS-[number].dbtmp +extern std::string OptionsFileName(const std::string& dbname, + uint64_t file_num); + +// Return a temp options file name given the "dbname" and file number. +// Format: OPTIONS-[number] +extern std::string TempOptionsFileName(const std::string& dbname, + uint64_t file_num); + // Return the name to use for a metadatabase. The result will be prefixed with // "dbname". extern std::string MetaDatabaseName(const std::string& dbname, diff --git a/db/options_file_test.cc b/db/options_file_test.cc new file mode 100644 index 000000000..808e492be --- /dev/null +++ b/db/options_file_test.cc @@ -0,0 +1,116 @@ +// Copyright (c) 2015, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef ROCKSDB_LITE +#include + +#include "db/db_impl.h" +#include "db/db_test_util.h" +#include "rocksdb/options.h" +#include "rocksdb/table.h" +#include "util/testharness.h" + +namespace rocksdb { +class OptionsFileTest : public testing::Test { + public: + OptionsFileTest() : dbname_(test::TmpDir() + "/options_file_test") {} + + std::string dbname_; +}; + +namespace { +void UpdateOptionsFiles(DB* db, + std::unordered_set* filename_history, + int* options_files_count) { + std::vector filenames; + db->GetEnv()->GetChildren(db->GetName(), &filenames); + uint64_t number; + FileType type; + *options_files_count = 0; + for (auto filename : filenames) { + if (ParseFileName(filename, &number, &type) && type == kOptionsFile) { + filename_history->insert(filename); + (*options_files_count)++; + } + } +} + +// Verify whether the current Options Files are the latest ones. +void VerifyOptionsFileName( + DB* db, const std::unordered_set& past_filenames) { + std::vector filenames; + std::unordered_set current_filenames; + db->GetEnv()->GetChildren(db->GetName(), &filenames); + uint64_t number; + FileType type; + for (auto filename : filenames) { + if (ParseFileName(filename, &number, &type) && type == kOptionsFile) { + current_filenames.insert(filename); + } + } + for (auto past_filename : past_filenames) { + if (current_filenames.find(past_filename) != current_filenames.end()) { + continue; + } + for (auto filename : current_filenames) { + ASSERT_GT(filename, past_filename); + } + } +} +} // namespace + +TEST_F(OptionsFileTest, NumberOfOptionsFiles) { + const int kReopenCount = 20; + Options opt; + opt.create_if_missing = true; + DestroyDB(dbname_, opt); + std::unordered_set filename_history; + DB* db; + for (int i = 0; i < kReopenCount; ++i) { + ASSERT_OK(DB::Open(opt, dbname_, &db)); + int num_options_files = 0; + UpdateOptionsFiles(db, &filename_history, &num_options_files); + ASSERT_GT(num_options_files, 0); + ASSERT_LE(num_options_files, 2); + // Make sure we always keep the latest option files. + VerifyOptionsFileName(db, filename_history); + delete db; + } +} + +TEST_F(OptionsFileTest, OptionsFileName) { + const uint64_t kOptionsFileNum = 12345; + uint64_t number; + FileType type; + + auto options_file_name = OptionsFileName("", kOptionsFileNum); + ASSERT_TRUE(ParseFileName(options_file_name, &number, &type, nullptr)); + ASSERT_EQ(type, kOptionsFile); + ASSERT_EQ(number, kOptionsFileNum); + + const uint64_t kTempOptionsFileNum = 54352; + auto temp_options_file_name = TempOptionsFileName("", kTempOptionsFileNum); + ASSERT_TRUE(ParseFileName(temp_options_file_name, &number, &type, nullptr)); + ASSERT_NE(temp_options_file_name.find(kTempFileNameSuffix), + std::string::npos); + ASSERT_EQ(type, kTempFile); + ASSERT_EQ(number, kTempOptionsFileNum); +} +} // namespace rocksdb + +int main(int argc, char** argv) { +#if !(defined NDEBUG) || !defined(OS_WIN) + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +#else + return 0; +#endif // !(defined NDEBUG) || !defined(OS_WIN) +} +#else +int main(int argc, char** argv) { + printf("Skipped as Options file is not supported in RocksDBLite.\n"); + return 0; +} +#endif // !ROCKSDB_LITE diff --git a/include/rocksdb/options.h b/include/rocksdb/options.h index 07678b667..f834a94ac 100644 --- a/include/rocksdb/options.h +++ b/include/rocksdb/options.h @@ -1172,6 +1172,13 @@ struct DBOptions { // currently. const WalFilter* wal_filter; #endif // ROCKSDB_LITE + + // If true, then DB::Open / CreateColumnFamily / DropColumnFamily + // / SetOptions will fail if options file is not detected or properly + // persisted. + // + // DEFAULT: false + bool fail_if_options_file_error; }; // Options to control the behavior of a database (passed to DB::Open) diff --git a/include/rocksdb/slice.h b/include/rocksdb/slice.h index ae3139cfd..3d39f3a04 100644 --- a/include/rocksdb/slice.h +++ b/include/rocksdb/slice.h @@ -73,6 +73,11 @@ class Slice { size_ -= n; } + void remove_suffix(size_t n) { + assert(n <= size()); + size_ -= n; + } + // Return a string that contains the copy of the referenced data. std::string ToString(bool hex = false) const; @@ -88,6 +93,11 @@ class Slice { (memcmp(data_, x.data_, x.size_) == 0)); } + bool ends_with(const Slice& x) const { + return ((size_ >= x.size_) && + (memcmp(data_ + size_ - x.size_, x.data_, x.size_) == 0)); + } + // Compare two slices and returns the first byte where they differ size_t difference_offset(const Slice& b) const; diff --git a/include/rocksdb/status.h b/include/rocksdb/status.h index e8e7970cc..f3f652af7 100644 --- a/include/rocksdb/status.h +++ b/include/rocksdb/status.h @@ -48,7 +48,7 @@ class Status { kAborted = 10, kBusy = 11, kExpired = 12, - kTryAgain = 13 + kTryAgain = 13, }; Code code() const { return code_; } diff --git a/src.mk b/src.mk index f357cdd93..02fed6ed7 100644 --- a/src.mk +++ b/src.mk @@ -206,6 +206,7 @@ TEST_BENCH_SOURCES = \ db/manual_compaction_test.cc \ db/memtablerep_bench.cc \ db/merge_test.cc \ + db/options_file_test.cc \ db/perf_context_test.cc \ db/plain_table_db_test.cc \ db/prefix_test.cc \ diff --git a/util/options.cc b/util/options.cc index 126e505cf..3a90cd860 100644 --- a/util/options.cc +++ b/util/options.cc @@ -259,12 +259,11 @@ DBOptions::DBOptions() enable_thread_tracking(false), delayed_write_rate(1024U * 1024U), skip_stats_update_on_db_open(false), - wal_recovery_mode(WALRecoveryMode::kTolerateCorruptedTailRecords) + wal_recovery_mode(WALRecoveryMode::kTolerateCorruptedTailRecords), #ifndef ROCKSDB_LITE - , - wal_filter(nullptr) + wal_filter(nullptr), #endif // ROCKSDB_LITE -{ + fail_if_options_file_error(false) { } DBOptions::DBOptions(const Options& options) @@ -323,12 +322,11 @@ DBOptions::DBOptions(const Options& options) delayed_write_rate(options.delayed_write_rate), skip_stats_update_on_db_open(options.skip_stats_update_on_db_open), wal_recovery_mode(options.wal_recovery_mode), - row_cache(options.row_cache) + row_cache(options.row_cache), #ifndef ROCKSDB_LITE - , - wal_filter(options.wal_filter) + wal_filter(options.wal_filter), #endif // ROCKSDB_LITE -{ + fail_if_options_file_error(options.fail_if_options_file_error) { } static const char* const access_hints[] = { diff --git a/util/options_helper.cc b/util/options_helper.cc index fa8dfb49f..3ff13ae31 100644 --- a/util/options_helper.cc +++ b/util/options_helper.cc @@ -1315,5 +1315,71 @@ Status GetTableFactoryFromMap( return Status::OK(); } +ColumnFamilyOptions BuildColumnFamilyOptions( + const Options& options, const MutableCFOptions& mutable_cf_options) { + ColumnFamilyOptions cf_opts(options); + + // Memtable related options + cf_opts.write_buffer_size = mutable_cf_options.write_buffer_size; + cf_opts.max_write_buffer_number = mutable_cf_options.max_write_buffer_number; + cf_opts.arena_block_size = mutable_cf_options.arena_block_size; + cf_opts.memtable_prefix_bloom_bits = + mutable_cf_options.memtable_prefix_bloom_bits; + cf_opts.memtable_prefix_bloom_probes = + mutable_cf_options.memtable_prefix_bloom_probes; + cf_opts.memtable_prefix_bloom_huge_page_tlb_size = + mutable_cf_options.memtable_prefix_bloom_huge_page_tlb_size; + cf_opts.max_successive_merges = mutable_cf_options.max_successive_merges; + cf_opts.filter_deletes = mutable_cf_options.filter_deletes; + cf_opts.inplace_update_num_locks = + mutable_cf_options.inplace_update_num_locks; + + // Compaction related options + cf_opts.disable_auto_compactions = + mutable_cf_options.disable_auto_compactions; + cf_opts.soft_rate_limit = mutable_cf_options.soft_rate_limit; + cf_opts.level0_file_num_compaction_trigger = + mutable_cf_options.level0_file_num_compaction_trigger; + cf_opts.level0_slowdown_writes_trigger = + mutable_cf_options.level0_slowdown_writes_trigger; + cf_opts.level0_stop_writes_trigger = + mutable_cf_options.level0_stop_writes_trigger; + cf_opts.max_grandparent_overlap_factor = + mutable_cf_options.max_grandparent_overlap_factor; + cf_opts.expanded_compaction_factor = + mutable_cf_options.expanded_compaction_factor; + cf_opts.source_compaction_factor = + mutable_cf_options.source_compaction_factor; + cf_opts.target_file_size_base = mutable_cf_options.target_file_size_base; + cf_opts.target_file_size_multiplier = + mutable_cf_options.target_file_size_multiplier; + cf_opts.max_bytes_for_level_base = + mutable_cf_options.max_bytes_for_level_base; + cf_opts.max_bytes_for_level_multiplier = + mutable_cf_options.max_bytes_for_level_multiplier; + + cf_opts.max_bytes_for_level_multiplier_additional.clear(); + for (auto value : + mutable_cf_options.max_bytes_for_level_multiplier_additional) { + cf_opts.max_bytes_for_level_multiplier_additional.emplace_back(value); + } + + cf_opts.verify_checksums_in_compaction = + mutable_cf_options.verify_checksums_in_compaction; + + // Misc options + cf_opts.max_sequential_skip_in_iterations = + mutable_cf_options.max_sequential_skip_in_iterations; + cf_opts.paranoid_file_checks = mutable_cf_options.paranoid_file_checks; + cf_opts.compaction_measure_io_stats = + mutable_cf_options.compaction_measure_io_stats; + + cf_opts.table_factory = options.table_factory; + // TODO(yhchiang): find some way to handle the following derived options + // * max_file_size + + return cf_opts; +} + #endif // !ROCKSDB_LITE } // namespace rocksdb diff --git a/util/options_helper.h b/util/options_helper.h index 4dead5507..c45f51d2f 100644 --- a/util/options_helper.h +++ b/util/options_helper.h @@ -64,6 +64,9 @@ Status GetTableFactoryFromMap( Status GetStringFromTableFactory(std::string* opts_str, const TableFactory* tf, const std::string& delimiter = "; "); +ColumnFamilyOptions BuildColumnFamilyOptions( + const Options& options, const MutableCFOptions& mutable_cf_options); + enum class OptionType { kBoolean, kInt, diff --git a/utilities/backupable/backupable_db_test.cc b/utilities/backupable/backupable_db_test.cc index cf5b77ac6..13917bd4a 100644 --- a/utilities/backupable/backupable_db_test.cc +++ b/utilities/backupable/backupable_db_test.cc @@ -1198,18 +1198,18 @@ TEST_F(BackupableDBTest, GarbageCollectionBeforeBackup) { OpenDBAndBackupEngine(true); env_->CreateDirIfMissing(backupdir_ + "/shared"); - std::string file_five = backupdir_ + "/shared/000005.sst"; + std::string file_five = backupdir_ + "/shared/000007.sst"; std::string file_five_contents = "I'm not really a sst file"; - // this depends on the fact that 00005.sst is the first file created by the DB + // this depends on the fact that 00007.sst is the first file created by the DB ASSERT_OK(file_manager_->WriteToFile(file_five, file_five_contents)); FillDB(db_.get(), 0, 100); - // backup overwrites file 000005.sst + // backup overwrites file 000007.sst ASSERT_TRUE(backup_engine_->CreateNewBackup(db_.get(), true).ok()); std::string new_file_five_contents; ASSERT_OK(ReadFileToString(env_, file_five, &new_file_five_contents)); - // file 000005.sst was overwritten + // file 000007.sst was overwritten ASSERT_TRUE(new_file_five_contents != file_five_contents); CloseDBAndBackupEngine(); -- GitLab