diff --git a/CMakeLists.txt b/CMakeLists.txt index 808f2a64a3507063ce974e184b7d878ffd3b3cb9..ad99eba364d1259a6a855bc3f764a8ae80e86986 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -329,6 +329,7 @@ set(TESTS db/memtable_list_test.cc db/merge_test.cc db/merge_helper_test.cc + db/options_file_test.cc db/perf_context_test.cc db/plain_table_db_test.cc db/prefix_test.cc diff --git a/HISTORY.md b/HISTORY.md index f4227078ce111a45bb05445992889c53f4b894e3..a54c13da4288c0e0081a164fa9ccce7b86b8952c 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -5,6 +5,7 @@ * Introduce CreateLoggerFromOptions(), this function create a Logger for provided DBOptions. * Add GetAggregatedIntProperty(), which returns the sum of the GetIntProperty of all the column families. * Add MemoryUtil in rocksdb/utilities/memory.h. It currently offers a way to get the memory usage by type from a list rocksdb instances. +* RocksDB will now persist options under the same directory as the RocksDB database on successful DB::Open, CreateColumnFamily, DropColumnFamily, and SetOptions. ### Public API Changes * CompactionFilter::Context includes information of Column Family ID * The need-compaction hint given by TablePropertiesCollector::NeedCompact() will be persistent and recoverable after DB recovery. This introduces a breaking format change. If you use this experimental feature, including NewCompactOnDeletionCollectorFactory() in the new version, you may not be able to directly downgrade the DB back to version 4.0 or lower. diff --git a/Makefile b/Makefile index bbb59891fb8df9fa540a0fbe9fd7ab244865e29a..aae039663b3b07b4fbe3ee0866c47dce698cfbd8 100644 --- a/Makefile +++ b/Makefile @@ -275,6 +275,7 @@ TESTS = \ memory_test \ merge_test \ merger_test \ + options_file_test \ redis_test \ reduce_levels_test \ plain_table_db_test \ @@ -892,6 +893,9 @@ merge_test: db/merge_test.o $(LIBOBJECTS) $(TESTHARNESS) merger_test: table/merger_test.o $(LIBOBJECTS) $(TESTHARNESS) $(AM_LINK) +options_file_test: db/options_file_test.o $(LIBOBJECTS) $(TESTHARNESS) + $(AM_LINK) + deletefile_test: db/deletefile_test.o $(LIBOBJECTS) $(TESTHARNESS) $(AM_LINK) diff --git a/db/column_family_test.cc b/db/column_family_test.cc index 938c4121ac8a92ae175a5d124ef87c624e820498..e776ae115af9c7b3e307b31a39341998906859ba 100644 --- a/db/column_family_test.cc +++ b/db/column_family_test.cc @@ -57,6 +57,7 @@ class ColumnFamilyTest : public testing::Test { env_ = new EnvCounter(Env::Default()); dbname_ = test::TmpDir() + "/column_family_test"; db_options_.create_if_missing = true; + db_options_.fail_if_options_file_error = true; db_options_.env = env_; DestroyDB(dbname_, Options(db_options_, column_family_options_)); } diff --git a/db/db_impl.cc b/db/db_impl.cc index 40b7ae81730d7836efb2f0c7d9884adf114f69e0..a250dbbcbbc969e1fe97cacecf7f5d8ed605e39b 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -84,6 +85,8 @@ #include "util/log_buffer.h" #include "util/logging.h" #include "util/mutexlock.h" +#include "util/options_helper.h" +#include "util/options_parser.h" #include "util/perf_context_imp.h" #include "util/stop_watch.h" #include "util/string_util.h" @@ -734,8 +737,12 @@ void DBImpl::PurgeObsoleteFiles(const JobContext& state) { // Also, SetCurrentFile creates a temp file when writing out new // manifest, which is equal to state.pending_manifest_file_number. We // should not delete that file + // + // TODO(yhchiang): carefully modify the third condition to safely + // remove the temp options files. keep = (sst_live_map.find(number) != sst_live_map.end()) || - (number == state.pending_manifest_file_number); + (number == state.pending_manifest_file_number) || + (to_delete.find(kOptionsFileNamePrefix) != std::string::npos); break; case kInfoLogFile: keep = true; @@ -747,6 +754,7 @@ void DBImpl::PurgeObsoleteFiles(const JobContext& state) { case kDBLockFile: case kIdentityFile: case kMetaDatabase: + case kOptionsFile: keep = true; break; } @@ -1922,6 +1930,19 @@ Status DBImpl::SetOptions(ColumnFamilyHandle* column_family, new_options = *cfd->GetLatestMutableCFOptions(); } } + if (s.ok()) { + Status persist_options_status = WriteOptionsFile(); + if (!persist_options_status.ok()) { + if (db_options_.fail_if_options_file_error) { + s = Status::IOError( + "SetOptions succeeded, but unable to persist options", + persist_options_status.ToString()); + } + Warn(db_options_.info_log, + "Unable to persist options in SetOptions() -- %s", + persist_options_status.ToString().c_str()); + } + } Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log, "SetOptions() on column family [%s], inputs:", @@ -3458,6 +3479,18 @@ Status DBImpl::CreateColumnFamily(const ColumnFamilyOptions& cf_options, // this is outside the mutex if (s.ok()) { + Status persist_options_status = WriteOptionsFile(); + if (!persist_options_status.ok()) { + if (db_options_.fail_if_options_file_error) { + s = Status::IOError( + "ColumnFamily has been created, but unable to persist" + "options in CreateColumnFamily()", + persist_options_status.ToString().c_str()); + } + Warn(db_options_.info_log, + "Unable to persist options in CreateColumnFamily() -- %s", + persist_options_status.ToString().c_str()); + } NewThreadStatusCfInfo( reinterpret_cast(*handle)->cfd()); } @@ -3515,6 +3548,18 @@ Status DBImpl::DropColumnFamily(ColumnFamilyHandle* column_family) { auto* mutable_cf_options = cfd->GetLatestMutableCFOptions(); max_total_in_memory_state_ -= mutable_cf_options->write_buffer_size * mutable_cf_options->max_write_buffer_number; + auto options_persist_status = WriteOptionsFile(); + if (!options_persist_status.ok()) { + if (db_options_.fail_if_options_file_error) { + s = Status::IOError( + "ColumnFamily has been dropped, but unable to persist " + "options in DropColumnFamily()", + options_persist_status.ToString().c_str()); + } + Warn(db_options_.info_log, + "Unable to persist options in DropColumnFamily() -- %s", + options_persist_status.ToString().c_str()); + } Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log, "Dropped column family with id %u\n", cfd->GetID()); @@ -4931,6 +4976,19 @@ Status DB::Open(const DBOptions& db_options, const std::string& dbname, impl); LogFlush(impl->db_options_.info_log); + auto persist_options_status = impl->WriteOptionsFile(); + if (!persist_options_status.ok()) { + if (db_options.fail_if_options_file_error) { + s = Status::IOError( + "DB::Open() failed --- Unable to persist Options file", + persist_options_status.ToString()); + } + Warn(impl->db_options_.info_log, + "Unable to persist options in DB::Open() -- %s", + persist_options_status.ToString().c_str()); + } + } + if (s.ok()) { *dbptr = impl; } else { for (auto* h : *handles) { @@ -4938,6 +4996,7 @@ Status DB::Open(const DBOptions& db_options, const std::string& dbname, } handles->clear(); delete impl; + *dbptr = nullptr; } return s; } @@ -5034,6 +5093,7 @@ Status DestroyDB(const std::string& dbname, const Options& options) { } } } + // ignore case where no archival directory is present. env->DeleteDir(archivedir); @@ -5045,6 +5105,114 @@ Status DestroyDB(const std::string& dbname, const Options& options) { return result; } +Status DBImpl::WriteOptionsFile() { +#ifndef ROCKSDB_LITE + std::string file_name; + Status s = WriteOptionsToTempFile(&file_name); + if (!s.ok()) { + return s; + } + s = RenameTempFileToOptionsFile(file_name); + return s; +#else + return Status::OK(); +#endif // !ROCKSDB_LITE +} + +Status DBImpl::WriteOptionsToTempFile(std::string* file_name) { +#ifndef ROCKSDB_LITE + std::vector cf_names; + std::vector cf_opts; + { + InstrumentedMutexLock l(&mutex_); + // This part requires mutex to protect the column family options + for (auto cfd : *versions_->GetColumnFamilySet()) { + if (cfd->IsDropped()) { + continue; + } + cf_names.push_back(cfd->GetName()); + cf_opts.push_back(BuildColumnFamilyOptions( + *cfd->options(), *cfd->GetLatestMutableCFOptions())); + } + } + *file_name = TempOptionsFileName(GetName(), versions_->NewFileNumber()); + + Status s = PersistRocksDBOptions(GetDBOptions(), cf_names, cf_opts, + *file_name, GetEnv()); + return s; +#else + return Status::OK(); +#endif // !ROCKSDB_LITE +} + +#ifndef ROCKSDB_LITE +namespace { +void DeleteOptionsFilesHelper(const std::map& filenames, + const size_t num_files_to_keep, + const std::shared_ptr& info_log, + Env* env) { + if (filenames.size() <= num_files_to_keep) { + return; + } + for (auto iter = std::next(filenames.begin(), num_files_to_keep); + iter != filenames.end(); ++iter) { + if (!env->DeleteFile(iter->second).ok()) { + Warn(info_log, "Unable to delete options file %s", iter->second.c_str()); + } + } +} +} // namespace +#endif // !ROCKSDB_LITE + +Status DBImpl::DeleteObsoleteOptionsFiles() { +#ifndef ROCKSDB_LITE + options_files_mutex_.AssertHeld(); + + std::vector filenames; + // use ordered map to store keep the filenames sorted from the newest + // to the oldest. + std::map options_filenames; + Status s; + s = GetEnv()->GetChildren(GetName(), &filenames); + if (!s.ok()) { + return s; + } + for (auto& filename : filenames) { + uint64_t file_number; + FileType type; + if (ParseFileName(filename, &file_number, &type) && type == kOptionsFile) { + options_filenames.insert( + {std::numeric_limits::max() - file_number, + GetName() + "/" + filename}); + } + } + + // Keeps the latest 2 Options file + const size_t kNumOptionsFilesKept = 2; + DeleteOptionsFilesHelper(options_filenames, kNumOptionsFilesKept, + db_options_.info_log, GetEnv()); + return Status::OK(); +#else + return Status::OK(); +#endif // !ROCKSDB_LITE +} + +Status DBImpl::RenameTempFileToOptionsFile(const std::string& file_name) { +#ifndef ROCKSDB_LITE + InstrumentedMutexLock l(&options_files_mutex_); + Status s; + std::string options_file_name = + OptionsFileName(GetName(), versions_->NewFileNumber()); + // Retry if the file name happen to conflict with an existing one. + s = GetEnv()->RenameFile(file_name, options_file_name); + + DeleteObsoleteOptionsFiles(); + return s; +#else + return Status::OK(); +#endif // !ROCKSDB_LITE +} + #if ROCKSDB_USING_THREAD_STATUS void DBImpl::NewThreadStatusCfInfo( diff --git a/db/db_impl.h b/db/db_impl.h index 3c538dac15f9d3a2233cd33204040573f7b3ebf6..b59cadfc191ad860550d4b3ff33b5fc6d27e2b5e 100644 --- a/db/db_impl.h +++ b/db/db_impl.h @@ -398,6 +398,13 @@ class DBImpl : public DB { SuperVersion* super_version, Arena* arena); + // The following options file related functions should not be + // called while DB mutex is held. + Status WriteOptionsFile(); + Status WriteOptionsToTempFile(std::string* file_name); + Status RenameTempFileToOptionsFile(const std::string& file_name); + Status DeleteObsoleteOptionsFiles(); + void NotifyOnFlushCompleted(ColumnFamilyData* cfd, FileMetaData* file_meta, const MutableCFOptions& mutable_cf_options, int job_id, TableProperties prop); @@ -552,8 +559,13 @@ class DBImpl : public DB { // Lock over the persistent DB state. Non-nullptr iff successfully acquired. FileLock* db_lock_; + // The mutex for options file related operations. + // NOTE: should never acquire options_file_mutex_ and mutex_ at the + // same time. + InstrumentedMutex options_files_mutex_; // State below is protected by mutex_ InstrumentedMutex mutex_; + std::atomic shutting_down_; // This condition variable is signaled on these conditions: // * whenever bg_compaction_scheduled_ goes down to 0 diff --git a/db/db_test_util.cc b/db/db_test_util.cc index 356398871849b01d3725b35593309e646d809aed..6cfb1781a6df523f3bf6e02fce235351dead00ba 100644 --- a/db/db_test_util.cc +++ b/db/db_test_util.cc @@ -368,6 +368,7 @@ Options DBTestBase::CurrentOptions( } options.env = env_; options.create_if_missing = true; + options.fail_if_options_file_error = true; return options; } diff --git a/db/filename.cc b/db/filename.cc index f57b178b9174ece8a0125268ea10431ac6fa4f99..32cd8758aac13fa913556ec547ad7617dc878984 100644 --- a/db/filename.cc +++ b/db/filename.cc @@ -21,6 +21,7 @@ #include "util/file_reader_writer.h" #include "util/logging.h" #include "util/stop_watch.h" +#include "util/string_util.h" #include "util/sync_point.h" namespace rocksdb { @@ -47,8 +48,9 @@ static size_t GetInfoLogPrefix(const std::string& path, char* dest, int len) { path[i] == '_'){ dest[write_idx++] = path[i]; } else { - if (i > 0) + if (i > 0) { dest[write_idx++] = '_'; + } } i++; } @@ -146,7 +148,7 @@ std::string LockFileName(const std::string& dbname) { } std::string TempFileName(const std::string& dbname, uint64_t number) { - return MakeFileName(dbname, number, "dbtmp"); + return MakeFileName(dbname, number, kTempFileNameSuffix.c_str()); } InfoLogPrefix::InfoLogPrefix(bool has_log_dir, @@ -186,6 +188,21 @@ std::string OldInfoLogFileName(const std::string& dbname, uint64_t ts, return log_dir + "/" + info_log_prefix.buf + ".old." + buf; } +std::string OptionsFileName(const std::string& dbname, uint64_t file_num) { + char buffer[256]; + snprintf(buffer, sizeof(buffer), "%s%06" PRIu64, + kOptionsFileNamePrefix.c_str(), file_num); + return dbname + "/" + buffer; +} + +std::string TempOptionsFileName(const std::string& dbname, uint64_t file_num) { + char buffer[256]; + snprintf(buffer, sizeof(buffer), "%s%06" PRIu64 ".%s", + kOptionsFileNamePrefix.c_str(), file_num, + kTempFileNameSuffix.c_str()); + return dbname + "/" + buffer; +} + std::string MetaDatabaseName(const std::string& dbname, uint64_t number) { char buf[100]; snprintf(buf, sizeof(buf), "/METADB-%llu", @@ -206,6 +223,8 @@ std::string IdentityFileName(const std::string& dbname) { // dbname/MANIFEST-[0-9]+ // dbname/[0-9]+.(log|sst) // dbname/METADB-[0-9]+ +// dbname/OPTIONS-[0-9]+ +// dbname/OPTIONS-[0-9]+.dbtmp // Disregards / at the beginning bool ParseFileName(const std::string& fname, uint64_t* number, @@ -268,6 +287,21 @@ bool ParseFileName(const std::string& fname, uint64_t* number, } *type = kMetaDatabase; *number = num; + } else if (rest.starts_with(kOptionsFileNamePrefix)) { + uint64_t ts_suffix; + bool is_temp_file = false; + rest.remove_prefix(kOptionsFileNamePrefix.size()); + const std::string kTempFileNameSuffixWithDot = + std::string(".") + kTempFileNameSuffix; + if (rest.ends_with(kTempFileNameSuffixWithDot)) { + rest.remove_suffix(kTempFileNameSuffixWithDot.size()); + is_temp_file = true; + } + if (!ConsumeDecimalNumber(&rest, &ts_suffix)) { + return false; + } + *number = ts_suffix; + *type = is_temp_file ? kTempFile : kOptionsFile; } else { // Avoid strtoull() to keep filename format independent of the // current locale @@ -302,7 +336,7 @@ bool ParseFileName(const std::string& fname, uint64_t* number, } else if (suffix == Slice(kRocksDbTFileExt) || suffix == Slice(kLevelDbTFileExt)) { *type = kTableFile; - } else if (suffix == Slice("dbtmp")) { + } else if (suffix == Slice(kTempFileNameSuffix)) { *type = kTempFile; } else { return false; diff --git a/db/filename.h b/db/filename.h index 926f027de9f76ce1e0160b6070c688718c4961b9..f7196c9f22d23b318ed082027c8ba1a3ba16f268 100644 --- a/db/filename.h +++ b/db/filename.h @@ -36,7 +36,8 @@ enum FileType { kTempFile, kInfoLogFile, // Either the current one, or an old one kMetaDatabase, - kIdentityFile + kIdentityFile, + kOptionsFile }; // Return the name of the log file with the specified number @@ -114,6 +115,19 @@ extern std::string OldInfoLogFileName(const std::string& dbname, uint64_t ts, const std::string& db_path = "", const std::string& log_dir = ""); +static const std::string kOptionsFileNamePrefix = "OPTIONS-"; +static const std::string kTempFileNameSuffix = "dbtmp"; + +// Return a options file name given the "dbname" and file number. +// Format: OPTIONS-[number].dbtmp +extern std::string OptionsFileName(const std::string& dbname, + uint64_t file_num); + +// Return a temp options file name given the "dbname" and file number. +// Format: OPTIONS-[number] +extern std::string TempOptionsFileName(const std::string& dbname, + uint64_t file_num); + // Return the name to use for a metadatabase. The result will be prefixed with // "dbname". extern std::string MetaDatabaseName(const std::string& dbname, diff --git a/db/options_file_test.cc b/db/options_file_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..808e492be615ce10928b979781096519ddd17ca6 --- /dev/null +++ b/db/options_file_test.cc @@ -0,0 +1,116 @@ +// Copyright (c) 2015, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef ROCKSDB_LITE +#include + +#include "db/db_impl.h" +#include "db/db_test_util.h" +#include "rocksdb/options.h" +#include "rocksdb/table.h" +#include "util/testharness.h" + +namespace rocksdb { +class OptionsFileTest : public testing::Test { + public: + OptionsFileTest() : dbname_(test::TmpDir() + "/options_file_test") {} + + std::string dbname_; +}; + +namespace { +void UpdateOptionsFiles(DB* db, + std::unordered_set* filename_history, + int* options_files_count) { + std::vector filenames; + db->GetEnv()->GetChildren(db->GetName(), &filenames); + uint64_t number; + FileType type; + *options_files_count = 0; + for (auto filename : filenames) { + if (ParseFileName(filename, &number, &type) && type == kOptionsFile) { + filename_history->insert(filename); + (*options_files_count)++; + } + } +} + +// Verify whether the current Options Files are the latest ones. +void VerifyOptionsFileName( + DB* db, const std::unordered_set& past_filenames) { + std::vector filenames; + std::unordered_set current_filenames; + db->GetEnv()->GetChildren(db->GetName(), &filenames); + uint64_t number; + FileType type; + for (auto filename : filenames) { + if (ParseFileName(filename, &number, &type) && type == kOptionsFile) { + current_filenames.insert(filename); + } + } + for (auto past_filename : past_filenames) { + if (current_filenames.find(past_filename) != current_filenames.end()) { + continue; + } + for (auto filename : current_filenames) { + ASSERT_GT(filename, past_filename); + } + } +} +} // namespace + +TEST_F(OptionsFileTest, NumberOfOptionsFiles) { + const int kReopenCount = 20; + Options opt; + opt.create_if_missing = true; + DestroyDB(dbname_, opt); + std::unordered_set filename_history; + DB* db; + for (int i = 0; i < kReopenCount; ++i) { + ASSERT_OK(DB::Open(opt, dbname_, &db)); + int num_options_files = 0; + UpdateOptionsFiles(db, &filename_history, &num_options_files); + ASSERT_GT(num_options_files, 0); + ASSERT_LE(num_options_files, 2); + // Make sure we always keep the latest option files. + VerifyOptionsFileName(db, filename_history); + delete db; + } +} + +TEST_F(OptionsFileTest, OptionsFileName) { + const uint64_t kOptionsFileNum = 12345; + uint64_t number; + FileType type; + + auto options_file_name = OptionsFileName("", kOptionsFileNum); + ASSERT_TRUE(ParseFileName(options_file_name, &number, &type, nullptr)); + ASSERT_EQ(type, kOptionsFile); + ASSERT_EQ(number, kOptionsFileNum); + + const uint64_t kTempOptionsFileNum = 54352; + auto temp_options_file_name = TempOptionsFileName("", kTempOptionsFileNum); + ASSERT_TRUE(ParseFileName(temp_options_file_name, &number, &type, nullptr)); + ASSERT_NE(temp_options_file_name.find(kTempFileNameSuffix), + std::string::npos); + ASSERT_EQ(type, kTempFile); + ASSERT_EQ(number, kTempOptionsFileNum); +} +} // namespace rocksdb + +int main(int argc, char** argv) { +#if !(defined NDEBUG) || !defined(OS_WIN) + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +#else + return 0; +#endif // !(defined NDEBUG) || !defined(OS_WIN) +} +#else +int main(int argc, char** argv) { + printf("Skipped as Options file is not supported in RocksDBLite.\n"); + return 0; +} +#endif // !ROCKSDB_LITE diff --git a/include/rocksdb/options.h b/include/rocksdb/options.h index 07678b667f2635a91c395d746d0a87117fc7eed0..f834a94aceadbbca810755c9bc209d00659b841c 100644 --- a/include/rocksdb/options.h +++ b/include/rocksdb/options.h @@ -1172,6 +1172,13 @@ struct DBOptions { // currently. const WalFilter* wal_filter; #endif // ROCKSDB_LITE + + // If true, then DB::Open / CreateColumnFamily / DropColumnFamily + // / SetOptions will fail if options file is not detected or properly + // persisted. + // + // DEFAULT: false + bool fail_if_options_file_error; }; // Options to control the behavior of a database (passed to DB::Open) diff --git a/include/rocksdb/slice.h b/include/rocksdb/slice.h index ae3139cfd6fea0f85bdcc3832c04933400457a37..3d39f3a046f8ff537292a71a81cfb2716bb9fc5b 100644 --- a/include/rocksdb/slice.h +++ b/include/rocksdb/slice.h @@ -73,6 +73,11 @@ class Slice { size_ -= n; } + void remove_suffix(size_t n) { + assert(n <= size()); + size_ -= n; + } + // Return a string that contains the copy of the referenced data. std::string ToString(bool hex = false) const; @@ -88,6 +93,11 @@ class Slice { (memcmp(data_, x.data_, x.size_) == 0)); } + bool ends_with(const Slice& x) const { + return ((size_ >= x.size_) && + (memcmp(data_ + size_ - x.size_, x.data_, x.size_) == 0)); + } + // Compare two slices and returns the first byte where they differ size_t difference_offset(const Slice& b) const; diff --git a/include/rocksdb/status.h b/include/rocksdb/status.h index e8e7970ccfdd3341e6f2b5f7e23df2ac7a09cc08..f3f652af74a2a914cb02a824654857729bdd1dc2 100644 --- a/include/rocksdb/status.h +++ b/include/rocksdb/status.h @@ -48,7 +48,7 @@ class Status { kAborted = 10, kBusy = 11, kExpired = 12, - kTryAgain = 13 + kTryAgain = 13, }; Code code() const { return code_; } diff --git a/src.mk b/src.mk index f357cdd93dd24697420be7419aa1fd1f93a855f9..02fed6ed75e8fdca6e923f25482451d36098c0c9 100644 --- a/src.mk +++ b/src.mk @@ -206,6 +206,7 @@ TEST_BENCH_SOURCES = \ db/manual_compaction_test.cc \ db/memtablerep_bench.cc \ db/merge_test.cc \ + db/options_file_test.cc \ db/perf_context_test.cc \ db/plain_table_db_test.cc \ db/prefix_test.cc \ diff --git a/util/options.cc b/util/options.cc index 126e505cfbbbb6bbbed5252aab4a3e2c61693ef5..3a90cd8609e07ce259e0c03f3f9fe06a16c4eb34 100644 --- a/util/options.cc +++ b/util/options.cc @@ -259,12 +259,11 @@ DBOptions::DBOptions() enable_thread_tracking(false), delayed_write_rate(1024U * 1024U), skip_stats_update_on_db_open(false), - wal_recovery_mode(WALRecoveryMode::kTolerateCorruptedTailRecords) + wal_recovery_mode(WALRecoveryMode::kTolerateCorruptedTailRecords), #ifndef ROCKSDB_LITE - , - wal_filter(nullptr) + wal_filter(nullptr), #endif // ROCKSDB_LITE -{ + fail_if_options_file_error(false) { } DBOptions::DBOptions(const Options& options) @@ -323,12 +322,11 @@ DBOptions::DBOptions(const Options& options) delayed_write_rate(options.delayed_write_rate), skip_stats_update_on_db_open(options.skip_stats_update_on_db_open), wal_recovery_mode(options.wal_recovery_mode), - row_cache(options.row_cache) + row_cache(options.row_cache), #ifndef ROCKSDB_LITE - , - wal_filter(options.wal_filter) + wal_filter(options.wal_filter), #endif // ROCKSDB_LITE -{ + fail_if_options_file_error(options.fail_if_options_file_error) { } static const char* const access_hints[] = { diff --git a/util/options_helper.cc b/util/options_helper.cc index fa8dfb49fb0eb28517ab5eb87300c560d7f7a337..3ff13ae31aeb4186231d06652cb0fe5031965b25 100644 --- a/util/options_helper.cc +++ b/util/options_helper.cc @@ -1315,5 +1315,71 @@ Status GetTableFactoryFromMap( return Status::OK(); } +ColumnFamilyOptions BuildColumnFamilyOptions( + const Options& options, const MutableCFOptions& mutable_cf_options) { + ColumnFamilyOptions cf_opts(options); + + // Memtable related options + cf_opts.write_buffer_size = mutable_cf_options.write_buffer_size; + cf_opts.max_write_buffer_number = mutable_cf_options.max_write_buffer_number; + cf_opts.arena_block_size = mutable_cf_options.arena_block_size; + cf_opts.memtable_prefix_bloom_bits = + mutable_cf_options.memtable_prefix_bloom_bits; + cf_opts.memtable_prefix_bloom_probes = + mutable_cf_options.memtable_prefix_bloom_probes; + cf_opts.memtable_prefix_bloom_huge_page_tlb_size = + mutable_cf_options.memtable_prefix_bloom_huge_page_tlb_size; + cf_opts.max_successive_merges = mutable_cf_options.max_successive_merges; + cf_opts.filter_deletes = mutable_cf_options.filter_deletes; + cf_opts.inplace_update_num_locks = + mutable_cf_options.inplace_update_num_locks; + + // Compaction related options + cf_opts.disable_auto_compactions = + mutable_cf_options.disable_auto_compactions; + cf_opts.soft_rate_limit = mutable_cf_options.soft_rate_limit; + cf_opts.level0_file_num_compaction_trigger = + mutable_cf_options.level0_file_num_compaction_trigger; + cf_opts.level0_slowdown_writes_trigger = + mutable_cf_options.level0_slowdown_writes_trigger; + cf_opts.level0_stop_writes_trigger = + mutable_cf_options.level0_stop_writes_trigger; + cf_opts.max_grandparent_overlap_factor = + mutable_cf_options.max_grandparent_overlap_factor; + cf_opts.expanded_compaction_factor = + mutable_cf_options.expanded_compaction_factor; + cf_opts.source_compaction_factor = + mutable_cf_options.source_compaction_factor; + cf_opts.target_file_size_base = mutable_cf_options.target_file_size_base; + cf_opts.target_file_size_multiplier = + mutable_cf_options.target_file_size_multiplier; + cf_opts.max_bytes_for_level_base = + mutable_cf_options.max_bytes_for_level_base; + cf_opts.max_bytes_for_level_multiplier = + mutable_cf_options.max_bytes_for_level_multiplier; + + cf_opts.max_bytes_for_level_multiplier_additional.clear(); + for (auto value : + mutable_cf_options.max_bytes_for_level_multiplier_additional) { + cf_opts.max_bytes_for_level_multiplier_additional.emplace_back(value); + } + + cf_opts.verify_checksums_in_compaction = + mutable_cf_options.verify_checksums_in_compaction; + + // Misc options + cf_opts.max_sequential_skip_in_iterations = + mutable_cf_options.max_sequential_skip_in_iterations; + cf_opts.paranoid_file_checks = mutable_cf_options.paranoid_file_checks; + cf_opts.compaction_measure_io_stats = + mutable_cf_options.compaction_measure_io_stats; + + cf_opts.table_factory = options.table_factory; + // TODO(yhchiang): find some way to handle the following derived options + // * max_file_size + + return cf_opts; +} + #endif // !ROCKSDB_LITE } // namespace rocksdb diff --git a/util/options_helper.h b/util/options_helper.h index 4dead5507ccf95ea7aaf2ee022c956ffc997d49f..c45f51d2fa3b3e3ba1ec4fe89d22d12c42730b0f 100644 --- a/util/options_helper.h +++ b/util/options_helper.h @@ -64,6 +64,9 @@ Status GetTableFactoryFromMap( Status GetStringFromTableFactory(std::string* opts_str, const TableFactory* tf, const std::string& delimiter = "; "); +ColumnFamilyOptions BuildColumnFamilyOptions( + const Options& options, const MutableCFOptions& mutable_cf_options); + enum class OptionType { kBoolean, kInt, diff --git a/utilities/backupable/backupable_db_test.cc b/utilities/backupable/backupable_db_test.cc index cf5b77ac60ac2716392b04b638293a03278ba32f..13917bd4a1339d046f6d2e7249c5c16b9e0c6043 100644 --- a/utilities/backupable/backupable_db_test.cc +++ b/utilities/backupable/backupable_db_test.cc @@ -1198,18 +1198,18 @@ TEST_F(BackupableDBTest, GarbageCollectionBeforeBackup) { OpenDBAndBackupEngine(true); env_->CreateDirIfMissing(backupdir_ + "/shared"); - std::string file_five = backupdir_ + "/shared/000005.sst"; + std::string file_five = backupdir_ + "/shared/000007.sst"; std::string file_five_contents = "I'm not really a sst file"; - // this depends on the fact that 00005.sst is the first file created by the DB + // this depends on the fact that 00007.sst is the first file created by the DB ASSERT_OK(file_manager_->WriteToFile(file_five, file_five_contents)); FillDB(db_.get(), 0, 100); - // backup overwrites file 000005.sst + // backup overwrites file 000007.sst ASSERT_TRUE(backup_engine_->CreateNewBackup(db_.get(), true).ok()); std::string new_file_five_contents; ASSERT_OK(ReadFileToString(env_, file_five, &new_file_five_contents)); - // file 000005.sst was overwritten + // file 000007.sst was overwritten ASSERT_TRUE(new_file_five_contents != file_five_contents); CloseDBAndBackupEngine();