From 06ad5dd2935707c790f0df4197bf9f5d90f55350 Mon Sep 17 00:00:00 2001 From: Peter Dillinger Date: Thu, 3 Sep 2020 23:49:27 -0700 Subject: [PATCH] Add file checksum to stress/crash test (#7343) Summary: This change has the crash test randomly select from a few file checksum implementations, or nullptr, for DB file_checksum_gen_factory. For compatibility across runs on same DB, each non-null factory can understand all the other functions, but the default changes. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7343 Test Plan: 'make blackbox_crash_test' for a while, including with some debug output to ensure code is being exercised. Reviewed By: zhichao-cao Differential Revision: D23494580 Pulled By: pdillinger fbshipit-source-id: 73bbc7ca32c1adaf619134c0c830f12894880b8a --- db_stress_tool/db_stress_common.cc | 105 ++++++++++++++++++++++++++ db_stress_tool/db_stress_common.h | 4 + db_stress_tool/db_stress_gflags.cc | 4 + db_stress_tool/db_stress_test_base.cc | 4 + tools/db_crashtest.py | 1 + 5 files changed, 118 insertions(+) diff --git a/db_stress_tool/db_stress_common.cc b/db_stress_tool/db_stress_common.cc index f86d75644..f1823302a 100644 --- a/db_stress_tool/db_stress_common.cc +++ b/db_stress_tool/db_stress_common.cc @@ -10,8 +10,12 @@ #ifdef GFLAGS #include "db_stress_tool/db_stress_common.h" + #include +#include "util/file_checksum_helper.h" +#include "util/xxhash.h" + ROCKSDB_NAMESPACE::DbStressEnvWrapper* db_stress_env = nullptr; #ifndef NDEBUG // If non-null, injects read error at a rate specified by the @@ -226,5 +230,106 @@ size_t GenerateValue(uint32_t rand, char* v, size_t max_sz) { v[value_sz] = '\0'; return value_sz; // the size of the value set. } + +namespace { + +class MyXXH64Checksum : public FileChecksumGenerator { + public: + explicit MyXXH64Checksum(bool big) : big_(big) { + state_ = XXH64_createState(); + XXH64_reset(state_, 0); + } + + virtual ~MyXXH64Checksum() override { XXH64_freeState(state_); } + + void Update(const char* data, size_t n) override { + XXH64_update(state_, data, n); + } + + void Finalize() override { + assert(str_.empty()); + uint64_t digest = XXH64_digest(state_); + // Store as little endian raw bytes + PutFixed64(&str_, digest); + if (big_) { + // Throw in some more data for stress testing (448 bits total) + PutFixed64(&str_, GetSliceHash64(str_)); + PutFixed64(&str_, GetSliceHash64(str_)); + PutFixed64(&str_, GetSliceHash64(str_)); + PutFixed64(&str_, GetSliceHash64(str_)); + PutFixed64(&str_, GetSliceHash64(str_)); + PutFixed64(&str_, GetSliceHash64(str_)); + } + } + + std::string GetChecksum() const override { + assert(!str_.empty()); + return str_; + } + + const char* Name() const override { + return big_ ? "MyBigChecksum" : "MyXXH64Checksum"; + } + + private: + bool big_; + XXH64_state_t* state_; + std::string str_; +}; + +class DbStressChecksumGenFactory : public FileChecksumGenFactory { + std::string default_func_name_; + + std::unique_ptr CreateFromFuncName( + const std::string& func_name) { + std::unique_ptr rv; + if (func_name == "FileChecksumCrc32c") { + rv.reset(new FileChecksumGenCrc32c(FileChecksumGenContext())); + } else if (func_name == "MyXXH64Checksum") { + rv.reset(new MyXXH64Checksum(false /* big */)); + } else if (func_name == "MyBigChecksum") { + rv.reset(new MyXXH64Checksum(true /* big */)); + } else { + // Should be a recognized function when we get here + assert(false); + } + return rv; + } + + public: + explicit DbStressChecksumGenFactory(const std::string& default_func_name) + : default_func_name_(default_func_name) {} + + std::unique_ptr CreateFileChecksumGenerator( + const FileChecksumGenContext& context) override { + if (context.requested_checksum_func_name.empty()) { + return CreateFromFuncName(default_func_name_); + } else { + return CreateFromFuncName(context.requested_checksum_func_name); + } + } + + const char* Name() const override { return "FileChecksumGenCrc32cFactory"; } +}; + +} // namespace + +std::shared_ptr GetFileChecksumImpl( + const std::string& name) { + // Translate from friendly names to internal names + std::string internal_name; + if (name == "crc32c") { + internal_name = "FileChecksumCrc32c"; + } else if (name == "xxh64") { + internal_name = "MyXXH64Checksum"; + } else if (name == "big") { + internal_name = "MyBigChecksum"; + } else { + assert(name.empty() || name == "none"); + return nullptr; + } + return std::make_shared(internal_name); +} + } // namespace ROCKSDB_NAMESPACE #endif // GFLAGS diff --git a/db_stress_tool/db_stress_common.h b/db_stress_tool/db_stress_common.h index 4224a0193..c6d9cf719 100644 --- a/db_stress_tool/db_stress_common.h +++ b/db_stress_tool/db_stress_common.h @@ -225,6 +225,7 @@ DECLARE_int32(verify_checksum_one_in); DECLARE_int32(verify_db_one_in); DECLARE_int32(continuous_verification_interval); DECLARE_int32(get_property_one_in); +DECLARE_string(file_checksum_impl); #ifndef ROCKSDB_LITE DECLARE_bool(use_blob_db); @@ -541,5 +542,8 @@ extern StressTest* CreateBatchedOpsStressTest(); extern StressTest* CreateNonBatchedOpsStressTest(); extern void InitializeHotKeyGenerator(double alpha); extern int64_t GetOneHotKeyID(double rand_seed, int64_t max_key); + +std::shared_ptr GetFileChecksumImpl( + const std::string& name); } // namespace ROCKSDB_NAMESPACE #endif // GFLAGS diff --git a/db_stress_tool/db_stress_gflags.cc b/db_stress_tool/db_stress_gflags.cc index 919a256c2..e83c0557f 100644 --- a/db_stress_tool/db_stress_gflags.cc +++ b/db_stress_tool/db_stress_gflags.cc @@ -728,4 +728,8 @@ DEFINE_bool(enable_compaction_filter, false, "If true, configures a compaction filter that returns a kRemove " "decision for deleted keys."); +DEFINE_string(file_checksum_impl, "none", + "Name of an implementation for file_checksum_gen_factory, or " + "\"none\" for null."); + #endif // GFLAGS diff --git a/db_stress_tool/db_stress_test_base.cc b/db_stress_tool/db_stress_test_base.cc index 7dcce08b2..4dcc921d6 100644 --- a/db_stress_tool/db_stress_test_base.cc +++ b/db_stress_tool/db_stress_test_base.cc @@ -1838,6 +1838,8 @@ void StressTest::PrintEnv() const { bottommost_compression.c_str()); std::string checksum = ChecksumTypeToString(checksum_type_e); fprintf(stdout, "Checksum type : %s\n", checksum.c_str()); + fprintf(stdout, "File checksum impl : %s\n", + FLAGS_file_checksum_impl.c_str()); fprintf(stdout, "Bloom bits / key : %s\n", FormatDoubleParam(FLAGS_bloom_bits).c_str()); fprintf(stdout, "Max subcompactions : %" PRIu64 "\n", @@ -1989,6 +1991,8 @@ void StressTest::Open() { FLAGS_max_write_batch_group_size_bytes; options_.level_compaction_dynamic_level_bytes = FLAGS_level_compaction_dynamic_level_bytes; + options_.file_checksum_gen_factory = + GetFileChecksumImpl(FLAGS_file_checksum_impl); } else { #ifdef ROCKSDB_LITE fprintf(stderr, "--options_file not supported in lite mode\n"); diff --git a/tools/db_crashtest.py b/tools/db_crashtest.py index 0f7239b67..2e2ecc1fc 100644 --- a/tools/db_crashtest.py +++ b/tools/db_crashtest.py @@ -61,6 +61,7 @@ default_params = { "enable_compaction_filter": lambda: random.choice([0, 0, 0, 1]), "expected_values_path": expected_values_file.name, "flush_one_in": 1000000, + "file_checksum_impl": lambda: random.choice(["none", "crc32c", "xxh64", "big"]), "get_live_files_one_in": 1000000, # Note: the following two are intentionally disabled as the corresponding # APIs are not guaranteed to succeed. -- GitLab