From 3160edfdc794e79195ad22a6620de5791be4bc6a Mon Sep 17 00:00:00 2001 From: anand76 Date: Fri, 20 Dec 2019 21:09:04 -0800 Subject: [PATCH] Generate variable length keys in db_stress (#6165) Summary: Currently, db_stress generates fixed length keys of 8 bytes. This patch adds the ability to generate variable length keys. Most of the db_stress code continues to work with a numeric key randomly generated, and the numeric key also acts as an index into the values_ array. The numeric key is mapped to a variable length string key in a deterministic way. Furthermore, the ordering is preserved. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6165 Test Plan: run make crash_test Differential Revision: D19204646 Pulled By: anand1976 fbshipit-source-id: d2d46a96615b4832a8be2a981f5913905f0e1ca7 --- db_stress_tool/cf_consistency_stress.cc | 8 +- db_stress_tool/db_stress_common.h | 111 ++++++++++++++++++++++-- db_stress_tool/db_stress_gflags.cc | 11 +++ db_stress_tool/db_stress_tool.cc | 34 ++++++++ db_stress_tool/no_batched_ops_stress.cc | 11 ++- tools/db_crashtest.py | 4 +- 6 files changed, 164 insertions(+), 15 deletions(-) diff --git a/db_stress_tool/cf_consistency_stress.cc b/db_stress_tool/cf_consistency_stress.cc index 0a8e1b7f6..ecacaf548 100644 --- a/db_stress_tool/cf_consistency_stress.cc +++ b/db_stress_tool/cf_consistency_stress.cc @@ -244,7 +244,9 @@ class CfConsistencyStressTest : public StressTest { std::string upper_bound; Slice ub_slice; ReadOptions ro_copy = readoptions; - if (thread->rand.OneIn(2) && GetNextPrefix(prefix, &upper_bound)) { + // Get the next prefix first and then see if we want to set upper bound. + // We'll use the next prefix in an assertion later on + if (GetNextPrefix(prefix, &upper_bound) && thread->rand.OneIn(2)) { ub_slice = Slice(upper_bound); ro_copy.iterate_upper_bound = &ub_slice; } @@ -252,13 +254,13 @@ class CfConsistencyStressTest : public StressTest { column_families_[rand_column_families[thread->rand.Next() % rand_column_families.size()]]; Iterator* iter = db_->NewIterator(ro_copy, cfh); - long count = 0; + unsigned long count = 0; for (iter->Seek(prefix); iter->Valid() && iter->key().starts_with(prefix); iter->Next()) { ++count; } assert(prefix_to_use == 0 || - count <= (static_cast(1) << ((8 - prefix_to_use) * 8))); + count <= GetPrefixKeyCount(prefix.ToString(), upper_bound)); Status s = iter->status(); if (s.ok()) { thread->stats.AddPrefixes(1, count); diff --git a/db_stress_tool/db_stress_common.h b/db_stress_tool/db_stress_common.h index 5867d8187..fe13a571b 100644 --- a/db_stress_tool/db_stress_common.h +++ b/db_stress_tool/db_stress_common.h @@ -82,6 +82,9 @@ DECLARE_uint64(seed); DECLARE_bool(read_only); DECLARE_int64(max_key); DECLARE_double(hot_key_alpha); +DECLARE_int32(max_key_len); +DECLARE_string(key_len_percent_dist); +DECLARE_int32(key_window_scale_factor); DECLARE_int32(column_families); DECLARE_string(options_file); DECLARE_int64(active_width); @@ -351,7 +354,7 @@ inline bool GetNextPrefix(const rocksdb::Slice& src, std::string* v) { #endif // convert long to a big-endian slice key -extern inline std::string Key(int64_t val) { +extern inline std::string GetStringFromInt(int64_t val) { std::string little_endian_key; std::string big_endian_key; PutFixed64(&little_endian_key, val); @@ -363,16 +366,110 @@ extern inline std::string Key(int64_t val) { return big_endian_key; } +// A struct for maintaining the parameters for generating variable length keys +struct KeyGenContext { + // Number of adjacent keys in one cycle of key lengths + uint64_t window; + // Number of keys of each possible length in a given window + std::vector weights; +}; +extern KeyGenContext key_gen_ctx; + +// Generate a variable length key string from the given int64 val. The +// order of the keys is preserved. The key could be anywhere from 8 to +// max_key_len * 8 bytes. +// The algorithm picks the length based on the +// offset of the val within a configured window and the distribution of the +// number of keys of various lengths in that window. For example, if x, y, x are +// the weights assigned to each possible key length, the keys generated would be +// - {0}...{x-1} +// {(x-1),0}..{(x-1),(y-1)},{(x-1),(y-1),0}..{(x-1),(y-1),(z-1)} and so on. +// Additionally, a trailer of 0-7 bytes could be appended. +extern inline std::string Key(int64_t val) { + uint64_t window = key_gen_ctx.window; + size_t levels = key_gen_ctx.weights.size(); + std::string key; + + for (size_t level = 0; level < levels; ++level) { + uint64_t weight = key_gen_ctx.weights[level]; + uint64_t offset = static_cast(val) % window; + uint64_t mult = static_cast(val) / window; + uint64_t pfx = mult * weight + (offset >= weight ? weight - 1 : offset); + key.append(GetStringFromInt(pfx)); + if (offset < weight) { + // Use the bottom 3 bits of offset as the number of trailing 'x's in the + // key. If the next key is going to be of the next level, then skip the + // trailer as it would break ordering. If the key length is already at max, + // skip the trailer. + if (offset < weight - 1 && level < levels - 1) { + size_t trailer_len = offset & 0x7; + key.append(trailer_len, 'x'); + } + break; + } + val = offset - weight; + window -= weight; + } + + return key; +} + +// Given a string key, map it to an index into the expected values buffer extern inline bool GetIntVal(std::string big_endian_key, uint64_t* key_p) { - unsigned int size_key = sizeof(*key_p); - assert(big_endian_key.size() == size_key); + size_t levels = key_gen_ctx.weights.size(); + size_t size_key = big_endian_key.size(); + std::vector prefixes; + + // Trim the key to multiple of 8 bytes + size_key &= ~7; + assert(size_key <= levels * sizeof(uint64_t)); + + // Pad with zeros to make it a multiple of 8. This function may be called + // with a prefix, in which case we return the first index that falls + // inside or outside that prefix, dependeing on whether the prefix is + // the start of upper bound of a scan + unsigned int pad = sizeof(uint64_t) - (size_key % sizeof(uint64_t)); + if (pad < sizeof(uint64_t)) { + big_endian_key.append(pad, '\0'); + } + std::string little_endian_key; little_endian_key.resize(size_key); - for (size_t i = 0; i < size_key; ++i) { - little_endian_key[i] = big_endian_key[size_key - 1 - i]; + for (size_t level = 0; level < size_key; level += sizeof(int64_t)) { + size_t start = level * sizeof(int64_t); + size_t end = (level + 1) * sizeof(int64_t); + for (size_t i = start; i < end; ++i) { + little_endian_key[i] = big_endian_key[end - 1 - i]; + } + Slice little_endian_slice = + Slice(&little_endian_key[start], sizeof(int64_t)); + uint64_t pfx; + if (!GetFixed64(&little_endian_slice, &pfx)) { + return false; + } + prefixes.emplace_back(pfx); + } + + uint64_t key = 0; + for (size_t i = 0; i < prefixes.size(); ++i) { + uint64_t pfx = prefixes[i]; + key += (pfx / key_gen_ctx.weights[i]) * key_gen_ctx.window + + pfx % key_gen_ctx.weights[i]; + } + *key_p = key; + return true; +} + +extern inline uint64_t GetPrefixKeyCount(const std::string& prefix, + const std::string& ub) { + uint64_t start = 0; + uint64_t end = 0; + + if (!GetIntVal(prefix, &start) || !GetIntVal(ub, &end)) { + return 0; } - Slice little_endian_slice = Slice(little_endian_key); - return GetFixed64(&little_endian_slice, key_p); + + return end - start; } extern inline std::string StringToHex(const std::string& str) { diff --git a/db_stress_tool/db_stress_gflags.cc b/db_stress_tool/db_stress_gflags.cc index 89a6d0433..02510f886 100644 --- a/db_stress_tool/db_stress_gflags.cc +++ b/db_stress_tool/db_stress_gflags.cc @@ -28,6 +28,17 @@ DEFINE_bool(read_only, false, "True if open DB in read-only mode during tests"); DEFINE_int64(max_key, 1 * KB * KB, "Max number of key/values to place in database"); +DEFINE_int32(max_key_len, 3, "Maximum length of a key in 8-byte units"); + +DEFINE_string(key_len_percent_dist, "", + "Percentages of keys of various lengths. For example, 1,30,69 " + "means 1% of keys are 8 bytes, 30% are 16 bytes, and 69% are " + "24 bytes. If not specified, it will be evenly distributed"); + +DEFINE_int32(key_window_scale_factor, 10, + "This value will be multiplied by 100 to come up with a window " + "size for varying the key length"); + DEFINE_int32(column_families, 10, "Number of column families"); DEFINE_double( diff --git a/db_stress_tool/db_stress_tool.cc b/db_stress_tool/db_stress_tool.cc index 61198c6f4..3ed1cf838 100644 --- a/db_stress_tool/db_stress_tool.cc +++ b/db_stress_tool/db_stress_tool.cc @@ -30,6 +30,8 @@ static std::shared_ptr env_guard; static std::shared_ptr env_wrapper_guard; } // namespace +KeyGenContext key_gen_ctx; + int db_stress_tool(int argc, char** argv) { SetUsageMessage(std::string("\nUSAGE:\n") + std::string(argv[0]) + " [OPTIONS]..."); @@ -197,6 +199,38 @@ int db_stress_tool(int argc, char** argv) { rocksdb_kill_odds = FLAGS_kill_random_test; rocksdb_kill_prefix_blacklist = SplitString(FLAGS_kill_prefix_blacklist); + unsigned int levels = FLAGS_max_key_len; + std::vector weights; + uint64_t scale_factor = FLAGS_key_window_scale_factor; + key_gen_ctx.window = scale_factor * 100; + if (!FLAGS_key_len_percent_dist.empty()) { + weights = SplitString(FLAGS_key_len_percent_dist); + if (weights.size() != levels) { + fprintf(stderr, + "Number of weights in key_len_dist should be equal to" + " max_key_len"); + exit(1); + } + + uint64_t total_weight = 0; + for (std::string& weight : weights) { + uint64_t val = std::stoull(weight); + key_gen_ctx.weights.emplace_back(val * scale_factor); + total_weight += val; + } + if (total_weight != 100) { + fprintf(stderr, "Sum of all weights in key_len_dist should be 100"); + exit(1); + } + } else { + uint64_t keys_per_level = key_gen_ctx.window / levels; + for (unsigned int level = 0; level < levels - 1; ++level) { + key_gen_ctx.weights.emplace_back(keys_per_level); + } + key_gen_ctx.weights.emplace_back(key_gen_ctx.window - + keys_per_level * (levels - 1)); + } + std::unique_ptr stress; if (FLAGS_test_cf_consistency) { stress.reset(CreateCfConsistencyStressTest()); diff --git a/db_stress_tool/no_batched_ops_stress.cc b/db_stress_tool/no_batched_ops_stress.cc index e3eac8796..da7796e36 100644 --- a/db_stress_tool/no_batched_ops_stress.cc +++ b/db_stress_tool/no_batched_ops_stress.cc @@ -52,12 +52,13 @@ class NonBatchedOpsStressTest : public StressTest { Slice k = keystr; Status s = iter->status(); if (iter->Valid()) { + Slice iter_key = iter->key(); if (iter->key().compare(k) > 0) { s = Status::NotFound(Slice()); } else if (iter->key().compare(k) == 0) { from_db = iter->value().ToString(); iter->Next(); - } else if (iter->key().compare(k) < 0) { + } else if (iter_key.compare(k) < 0) { VerificationAbort(shared, "An out of range key was found", static_cast(cf), i); } @@ -197,19 +198,21 @@ class NonBatchedOpsStressTest : public StressTest { std::string upper_bound; Slice ub_slice; ReadOptions ro_copy = read_opts; - if (thread->rand.OneIn(2) && GetNextPrefix(prefix, &upper_bound)) { + // Get the next prefix first and then see if we want to set upper bound. + // We'll use the next prefix in an assertion later on + if (GetNextPrefix(prefix, &upper_bound) && thread->rand.OneIn(2)) { // For half of the time, set the upper bound to the next prefix ub_slice = Slice(upper_bound); ro_copy.iterate_upper_bound = &ub_slice; } Iterator* iter = db_->NewIterator(ro_copy, cfh); - long count = 0; + unsigned long count = 0; for (iter->Seek(prefix); iter->Valid() && iter->key().starts_with(prefix); iter->Next()) { ++count; } - assert(count <= (static_cast(1) << ((8 - FLAGS_prefix_size) * 8))); + assert(count <= GetPrefixKeyCount(prefix.ToString(), upper_bound)); Status s = iter->status(); if (iter->status().ok()) { thread->stats.AddPrefixes(1, count); diff --git a/tools/db_crashtest.py b/tools/db_crashtest.py index 3319d81f6..3fe72982e 100644 --- a/tools/db_crashtest.py +++ b/tools/db_crashtest.py @@ -107,7 +107,9 @@ default_params = { # "level_compaction_dynamic_level_bytes" : True, "verify_checksum_one_in": 1000000, "verify_db_one_in": 100000, - "continuous_verification_interval" : 0 + "continuous_verification_interval" : 0, + "max_key_len": 3, + "key_len_percent_dist": "1,30,69" } _TEST_DIR_ENV_VAR = 'TEST_TMPDIR' -- GitLab