From a6538571788f52b87a3ca83667b8634ee4950f54 Mon Sep 17 00:00:00 2001 From: Peter Dillinger Date: Tue, 10 Dec 2019 15:45:25 -0800 Subject: [PATCH] Add PauseBackgroundWork() to db_stress (#6148) Summary: Worker thread will occasionally call PauseBackgroundWork(), briefly sleep (to avoid stalling itself) and then call ContinueBackgroundWork(). Pull Request resolved: https://github.com/facebook/rocksdb/pull/6148 Test Plan: some running of 'make blackbox_crash_test' with temporary printf output to confirm code occasionally reached. Differential Revision: D18913886 Pulled By: pdillinger fbshipit-source-id: ae9356a803390929f3165dfb6a00194692ba92be --- db_stress_tool/db_stress_common.h | 1 + db_stress_tool/db_stress_gflags.cc | 4 ++++ db_stress_tool/db_stress_test_base.cc | 22 ++++++++++++++++++++++ tools/db_crashtest.py | 1 + 4 files changed, 28 insertions(+) diff --git a/db_stress_tool/db_stress_common.h b/db_stress_tool/db_stress_common.h index fb9b5f9f0..8bb9667de 100644 --- a/db_stress_tool/db_stress_common.h +++ b/db_stress_tool/db_stress_common.h @@ -168,6 +168,7 @@ DECLARE_int32(ingest_external_file_width); DECLARE_int32(compact_files_one_in); DECLARE_int32(compact_range_one_in); DECLARE_int32(flush_one_in); +DECLARE_int32(pause_background_one_in); DECLARE_int32(compact_range_width); DECLARE_int32(acquire_snapshot_one_in); DECLARE_bool(compare_full_db_state_snapshot); diff --git a/db_stress_tool/db_stress_gflags.cc b/db_stress_tool/db_stress_gflags.cc index e1f01b14d..a4198be6b 100644 --- a/db_stress_tool/db_stress_gflags.cc +++ b/db_stress_tool/db_stress_gflags.cc @@ -401,6 +401,10 @@ DEFINE_int32(flush_one_in, 0, "If non-zero, then Flush() will be called once for every N ops " "on average. 0 indicates calls to Flush() are disabled."); +DEFINE_int32(pause_background_one_in, 0, + "If non-zero, then PauseBackgroundWork()+Continue will be called " + "once for every N ops on average. 0 disables."); + DEFINE_int32(compact_range_width, 10000, "The width of the ranges passed to CompactRange()."); diff --git a/db_stress_tool/db_stress_test_base.cc b/db_stress_tool/db_stress_test_base.cc index b92100334..2fa150dc8 100644 --- a/db_stress_tool/db_stress_test_base.cc +++ b/db_stress_tool/db_stress_test_base.cc @@ -600,6 +600,28 @@ void StressTest::OperateDb(ThreadState* thread) { } } + if (FLAGS_pause_background_one_in > 0 && + thread->rand.OneIn(FLAGS_pause_background_one_in)) { + Status status = db_->PauseBackgroundWork(); + if (!status.ok()) { + VerificationAbort(shared, "PauseBackgroundWork status not OK", + status); + } + // To avoid stalling/deadlocking ourself in this thread, just + // sleep here during pause and let other threads do db operations. + // Sleep up to ~16 seconds (2**24 microseconds), but very skewed + // toward short pause. (1 chance in 25 of pausing >= 1s; + // 1 chance in 625 of pausing full 16s.) + int pwr2_micros = + std::min(thread->rand.Uniform(25), thread->rand.Uniform(25)); + FLAGS_env->SleepForMicroseconds(1 << pwr2_micros); + status = db_->ContinueBackgroundWork(); + if (!status.ok()) { + VerificationAbort(shared, "ContinueBackgroundWork status not OK", + status); + } + } + std::vector rand_keys = GenerateKeys(rand_key); if (FLAGS_ingest_external_file_one_in > 0 && diff --git a/tools/db_crashtest.py b/tools/db_crashtest.py index 9f9c83a6b..4638d52c7 100644 --- a/tools/db_crashtest.py +++ b/tools/db_crashtest.py @@ -52,6 +52,7 @@ default_params = { "nooverwritepercent": 1, "open_files": lambda : random.choice([-1, 500000]), "partition_filters": lambda: random.randint(0, 1), + "pause_background_one_in": 1000000, "prefixpercent": 5, "progress_reports": 0, "readpercent": 45, -- GitLab