From d781da816410add0874116ac388d1c58c0a063e8 Mon Sep 17 00:00:00 2001 From: Yueh-Hsuan Chiang Date: Thu, 12 Nov 2015 16:52:51 -0800 Subject: [PATCH] Add CheckOptionsCompatibility() API to options_util Summary: Add CheckOptionsCompatibility() API to options_util that returns Status::OK if the input DBOptions and ColumnFamilyDescriptors are compatible with the latest options stored in the specified DB path. Test Plan: Added tests in options_util_test Reviewers: igor, anthony, IslamAbdelRahman, rven, sdong Reviewed By: sdong Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D50649 --- HISTORY.md | 1 + include/rocksdb/utilities/options_util.h | 15 ++ utilities/options/options_util.cc | 23 +++ utilities/options/options_util_test.cc | 193 ++++++++++++++++++++++- 4 files changed, 225 insertions(+), 7 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index a6a4018db..7ba14c9ef 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -7,6 +7,7 @@ * Add MemoryUtil in rocksdb/utilities/memory.h. It currently offers a way to get the memory usage by type from a list rocksdb instances. * RocksDB will now persist options under the same directory as the RocksDB database on successful DB::Open, CreateColumnFamily, DropColumnFamily, and SetOptions. * Introduce LoadLatestOptions() in rocksdb/utilities/options_util.h. This function can construct the latest DBOptions / ColumnFamilyOptions used by the specified RocksDB intance. +* Introduce CheckOptionsCompatibility() in rocksdb/utilities/options_util.h. This function checks whether the input set of options is able to open the specified DB successfully. ### Public API Changes * CompactionFilter::Context includes information of Column Family ID * The need-compaction hint given by TablePropertiesCollector::NeedCompact() will be persistent and recoverable after DB recovery. This introduces a breaking format change. If you use this experimental feature, including NewCompactOnDeletionCollectorFactory() in the new version, you may not be able to directly downgrade the DB back to version 4.0 or lower. diff --git a/include/rocksdb/utilities/options_util.h b/include/rocksdb/utilities/options_util.h index e1b4c3fda..a2c09fe31 100644 --- a/include/rocksdb/utilities/options_util.h +++ b/include/rocksdb/utilities/options_util.h @@ -61,5 +61,20 @@ Status LoadOptionsFromFile(const std::string& options_file_name, Env* env, Status GetLatestOptionsFileName(const std::string& dbpath, Env* env, std::string* options_file_name); +// Returns Status::OK if the input DBOptions and ColumnFamilyDescriptors +// are compatible with the latest options stored in the specified DB path. +// +// If the return status is non-ok, it means the specified RocksDB instance +// might not be correctly opened with the input set of options. Currently, +// changing one of the following options will fail the compatibility check: +// +// * comparator +// * prefix_extractor +// * table_factory +// * merge_operator +Status CheckOptionsCompatibility( + const std::string& dbpath, Env* env, const DBOptions& db_options, + const std::vector& cf_descs); + } // namespace rocksdb #endif // !ROCKSDB_LITE diff --git a/utilities/options/options_util.cc b/utilities/options/options_util.cc index 6a595c3d8..1c6a068ac 100644 --- a/utilities/options/options_util.cc +++ b/utilities/options/options_util.cc @@ -72,5 +72,28 @@ Status LoadLatestOptions(const std::string& dbpath, Env* env, db_options, cf_descs); } +Status CheckOptionsCompatibility( + const std::string& dbpath, Env* env, const DBOptions& db_options, + const std::vector& cf_descs) { + std::string options_file_name; + Status s = GetLatestOptionsFileName(dbpath, env, &options_file_name); + if (!s.ok()) { + return s; + } + + std::vector cf_names; + std::vector cf_opts; + for (const auto& cf_desc : cf_descs) { + cf_names.push_back(cf_desc.name); + cf_opts.push_back(cf_desc.options); + } + + const OptionsSanityCheckLevel kDefaultLevel = kSanityLevelLooselyCompatible; + + return RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( + db_options, cf_names, cf_opts, dbpath + "/" + options_file_name, env, + kDefaultLevel); +} + } // namespace rocksdb #endif // !ROCKSDB_LITE diff --git a/utilities/options/options_util_test.cc b/utilities/options/options_util_test.cc index a017f5600..c6d8cdb5f 100644 --- a/utilities/options/options_util_test.cc +++ b/utilities/options/options_util_test.cc @@ -13,6 +13,8 @@ #include #include +#include "rocksdb/db.h" +#include "rocksdb/table.h" #include "rocksdb/utilities/options_util.h" #include "util/options_parser.h" #include "util/random.h" @@ -30,10 +32,15 @@ DEFINE_bool(enable_print, false, "Print options generated to console."); namespace rocksdb { class OptionsUtilTest : public testing::Test { public: - OptionsUtilTest() { env_.reset(new test::StringEnv(Env::Default())); } + OptionsUtilTest() : rnd_(0xFB) { + env_.reset(new test::StringEnv(Env::Default())); + dbname_ = test::TmpDir() + "/options_util_test"; + } protected: std::unique_ptr env_; + std::string dbname_; + Random rnd_; }; bool IsBlockBasedTableFactory(TableFactory* tf) { @@ -42,17 +49,16 @@ bool IsBlockBasedTableFactory(TableFactory* tf) { TEST_F(OptionsUtilTest, SaveAndLoad) { const size_t kCFCount = 5; - Random rnd(0xFB); DBOptions db_opt; std::vector cf_names; std::vector cf_opts; - test::RandomInitDBOptions(&db_opt, &rnd); + test::RandomInitDBOptions(&db_opt, &rnd_); for (size_t i = 0; i < kCFCount; ++i) { cf_names.push_back(i == 0 ? kDefaultColumnFamilyName - : test::RandomName(&rnd, 10)); + : test::RandomName(&rnd_, 10)); cf_opts.emplace_back(); - test::RandomInitCFOptions(&cf_opts.back(), &rnd); + test::RandomInitCFOptions(&cf_opts.back(), &rnd_); } const std::string kFileName = "OPTIONS-123456"; @@ -64,7 +70,7 @@ TEST_F(OptionsUtilTest, SaveAndLoad) { &loaded_cf_descs)); ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(db_opt, loaded_db_opt)); - test::RandomInitDBOptions(&db_opt, &rnd); + test::RandomInitDBOptions(&db_opt, &rnd_); ASSERT_NOK(RocksDBOptionsParser::VerifyDBOptions(db_opt, loaded_db_opt)); for (size_t i = 0; i < kCFCount; ++i) { @@ -76,7 +82,7 @@ TEST_F(OptionsUtilTest, SaveAndLoad) { cf_opts[i].table_factory.get(), loaded_cf_descs[i].options.table_factory.get())); } - test::RandomInitCFOptions(&cf_opts[i], &rnd); + test::RandomInitCFOptions(&cf_opts[i], &rnd_); ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions( cf_opts[i], loaded_cf_descs[i].options)); } @@ -88,6 +94,179 @@ TEST_F(OptionsUtilTest, SaveAndLoad) { } } +namespace { +class DummyTableFactory : public TableFactory { + public: + DummyTableFactory() {} + virtual ~DummyTableFactory() {} + + virtual const char* Name() const { return "DummyTableFactory"; } + + virtual Status NewTableReader(const TableReaderOptions& table_reader_options, + unique_ptr&& file, + uint64_t file_size, + unique_ptr* table_reader) const { + return Status::NotSupported(); + } + + virtual TableBuilder* NewTableBuilder( + const TableBuilderOptions& table_builder_options, + uint32_t column_family_id, WritableFileWriter* file) const { + return nullptr; + } + + virtual Status SanitizeOptions(const DBOptions& db_opts, + const ColumnFamilyOptions& cf_opts) const { + return Status::NotSupported(); + } + + virtual std::string GetPrintableTableOptions() const { return ""; } +}; + +class DummyMergeOperator : public MergeOperator { + public: + DummyMergeOperator() {} + virtual ~DummyMergeOperator() {} + + virtual bool FullMerge(const Slice& key, const Slice* existing_value, + const std::deque& operand_list, + std::string* new_value, Logger* logger) const { + return false; + } + + virtual bool PartialMergeMulti(const Slice& key, + const std::deque& operand_list, + std::string* new_value, Logger* logger) const { + return false; + } + + virtual const char* Name() const { return "DummyMergeOperator"; } +}; + +class DummySliceTransform : public SliceTransform { + public: + DummySliceTransform() {} + virtual ~DummySliceTransform() {} + + // Return the name of this transformation. + virtual const char* Name() const { return "DummySliceTransform"; } + + // transform a src in domain to a dst in the range + virtual Slice Transform(const Slice& src) const { return src; } + + // determine whether this is a valid src upon the function applies + virtual bool InDomain(const Slice& src) const { return false; } + + // determine whether dst=Transform(src) for some src + virtual bool InRange(const Slice& dst) const { return false; } +}; + +} // namespace + +TEST_F(OptionsUtilTest, SanityCheck) { + DBOptions db_opt; + std::vector cf_descs; + const size_t kCFCount = 5; + for (size_t i = 0; i < kCFCount; ++i) { + cf_descs.emplace_back(); + cf_descs.back().name = + (i == 0) ? kDefaultColumnFamilyName : test::RandomName(&rnd_, 10); + + cf_descs.back().options.table_factory.reset(NewBlockBasedTableFactory()); + cf_descs.back().options.prefix_extractor.reset( + test::RandomSliceTransform(&rnd_)); + cf_descs.back().options.merge_operator.reset( + test::RandomMergeOperator(&rnd_)); + } + + db_opt.create_missing_column_families = true; + db_opt.create_if_missing = true; + + DestroyDB(dbname_, Options(db_opt, cf_descs[0].options)); + DB* db; + std::vector handles; + // open and persist the options + ASSERT_OK(DB::Open(db_opt, dbname_, cf_descs, &handles, &db)); + + // close the db + for (auto* handle : handles) { + delete handle; + } + delete db; + + // perform sanity check + ASSERT_OK( + CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs)); + + ASSERT_GE(kCFCount, 5); + // merge operator + { + std::shared_ptr merge_op = + cf_descs[0].options.merge_operator; + + ASSERT_NE(merge_op.get(), nullptr); + cf_descs[0].options.merge_operator.reset(); + ASSERT_NOK( + CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs)); + + cf_descs[0].options.merge_operator.reset(new DummyMergeOperator()); + ASSERT_NOK( + CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs)); + + cf_descs[0].options.merge_operator = merge_op; + ASSERT_OK( + CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs)); + } + + // prefix extractor + { + std::shared_ptr prefix_extractor = + cf_descs[1].options.prefix_extractor; + + ASSERT_NE(prefix_extractor, nullptr); + cf_descs[1].options.prefix_extractor.reset(); + ASSERT_NOK( + CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs)); + + cf_descs[1].options.prefix_extractor.reset(new DummySliceTransform()); + ASSERT_NOK( + CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs)); + + cf_descs[1].options.prefix_extractor = prefix_extractor; + ASSERT_OK( + CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs)); + } + + // comparator + { + test::SimpleSuffixReverseComparator comparator; + + auto* prev_comparator = cf_descs[2].options.comparator; + cf_descs[2].options.comparator = &comparator; + ASSERT_NOK( + CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs)); + + cf_descs[2].options.comparator = prev_comparator; + ASSERT_OK( + CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs)); + } + + // table factory + { + std::shared_ptr table_factory = + cf_descs[3].options.table_factory; + + ASSERT_NE(table_factory, nullptr); + cf_descs[3].options.table_factory.reset(new DummyTableFactory()); + ASSERT_NOK( + CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs)); + + cf_descs[3].options.table_factory = table_factory; + ASSERT_OK( + CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs)); + } +} + } // namespace rocksdb int main(int argc, char** argv) { -- GitLab