db_stress support for range deletions

Summary:
made db_stress capable of adding range deletions to its db and verifying their correctness. i'll make db_crashtest.py use this option later once the collapsing optimization (https://github.com/facebook/rocksdb/pull/1614) is committed because currently it slows down the test too much.
Closes https://github.com/facebook/rocksdb/pull/1625

Differential Revision: D4293939

Pulled By: ajkr

fbshipit-source-id: d3beb3a

tools/db_stress.cc

@@ -306,6 +306,9 @@ DEFINE_uint64(max_bytes_for_level_base, 256 * KB, "Max bytes for level-1");
 DEFINE_double(max_bytes_for_level_multiplier, 2,
               "A multiplier to compute max bytes for level-N (N >= 2)");
 
+DEFINE_int32(range_deletion_width, 10,
+             "The width of the range deletion intervals.");
+
 // Temporarily disable this to allows it to detect new bugs
 DEFINE_int32(compact_files_one_in, 0,
              "If non-zero, then CompactFiles() will be called one for every N "
@@ -341,6 +344,12 @@ DEFINE_int32(delpercent, 15,
 static const bool FLAGS_delpercent_dummy __attribute__((unused)) =
     RegisterFlagValidator(&FLAGS_delpercent, &ValidateInt32Percent);
 
+DEFINE_int32(delrangepercent, 0,
+             "Ratio of range deletions to total workload (expressed as a "
+             "percentage). Cannot be used with test_batches_snapshots");
+static const bool FLAGS_delrangepercent_dummy __attribute__((unused)) =
+    RegisterFlagValidator(&FLAGS_delrangepercent, &ValidateInt32Percent);
+
 DEFINE_int32(nooverwritepercent, 60,
              "Ratio of keys without overwrite to total workload (expressed as "
              " a percentage)");
@@ -499,6 +508,8 @@ class Stats {
   long iterator_size_sums_;
   long founds_;
   long iterations_;
+  long range_deletions_;
+  long covered_by_range_deletions_;
   long errors_;
   long num_compact_files_succeed_;
   long num_compact_files_failed_;
@@ -522,6 +533,8 @@ class Stats {
     iterator_size_sums_ = 0;
     founds_ = 0;
     iterations_ = 0;
+    range_deletions_ = 0;
+    covered_by_range_deletions_ = 0;
     errors_ = 0;
     bytes_ = 0;
     seconds_ = 0;
@@ -543,6 +556,8 @@ class Stats {
     iterator_size_sums_ += other.iterator_size_sums_;
     founds_ += other.founds_;
     iterations_ += other.iterations_;
+    range_deletions_ += other.range_deletions_;
+    covered_by_range_deletions_ = other.covered_by_range_deletions_;
     errors_ += other.errors_;
     bytes_ += other.bytes_;
     seconds_ += other.seconds_;
@@ -608,6 +623,14 @@ class Stats {
 
   void AddSingleDeletes(size_t n) { single_deletes_ += n; }
 
+  void AddRangeDeletions(int n) {
+    range_deletions_ += n;
+  }
+
+  void AddCoveredByRangeDeletions(int n) {
+    covered_by_range_deletions_ += n;
+  }
+
   void AddErrors(int n) {
     errors_ += n;
   }
@@ -643,6 +666,10 @@ class Stats {
     fprintf(stdout, "%-12s: Iterator size sum is %ld\n", "",
             iterator_size_sums_);
     fprintf(stdout, "%-12s: Iterated %ld times\n", "", iterations_);
+    fprintf(stdout, "%-12s: Deleted %ld key-ranges\n", "", range_deletions_);
+    fprintf(stdout, "%-12s: Range deletions covered %ld keys\n", "",
+            covered_by_range_deletions_);
+
     fprintf(stdout, "%-12s: Got errors %ld times\n", "", errors_);
     fprintf(stdout, "%-12s: %ld CompactFiles() succeed\n", "",
             num_compact_files_succeed_);
@@ -824,6 +851,17 @@ class SharedState {
 
   void SingleDelete(int cf, int64_t key) { values_[cf][key] = SENTINEL; }
 
+  int DeleteRange(int cf, int64_t begin_key, int64_t end_key) {
+    int covered = 0;
+    for (int64_t key = begin_key; key < end_key; ++key) {
+      if (values_[cf][key] != SENTINEL) {
+        ++covered;
+      }
+      values_[cf][key] = SENTINEL;
+    }
+    return covered;
+  }
+
   bool AllowsOverwrite(int cf, int64_t key) {
     return no_overwrite_ids_[cf].find(key) == no_overwrite_ids_[cf].end();
   }
@@ -1550,6 +1588,7 @@ class StressTest {
     const int prefixBound = (int)FLAGS_readpercent + (int)FLAGS_prefixpercent;
     const int writeBound = prefixBound + (int)FLAGS_writepercent;
     const int delBound = writeBound + (int)FLAGS_delpercent;
+    const int delRangeBound = delBound + (int)FLAGS_delrangepercent;
 
     thread->stats.Start();
     for (uint64_t i = 0; i < FLAGS_ops_per_thread; i++) {
@@ -1814,6 +1853,47 @@ class StressTest {
         } else {
           MultiDelete(thread, write_opts, column_family, key);
         }
+      } else if (delBound <= prob_op && prob_op < delRangeBound) {
+        // OPERATION delete range
+        if (!FLAGS_test_batches_snapshots) {
+          std::vector<std::unique_ptr<MutexLock>> range_locks;
+          // delete range does not respect disallowed overwrites. the keys for
+          // which overwrites are disallowed are randomly distributed so it
+          // could be expensive to find a range where each key allows
+          // overwrites.
+          if (rand_key > max_key - FLAGS_range_deletion_width) {
+            l.reset();
+            rand_key = thread->rand.Next() %
+                       (max_key - FLAGS_range_deletion_width + 1);
+            range_locks.emplace_back(new MutexLock(
+                shared->GetMutexForKey(rand_column_family, rand_key)));
+          } else {
+            range_locks.emplace_back(std::move(l));
+          }
+          for (int j = 1; j < FLAGS_range_deletion_width; ++j) {
+            if (((rand_key + j) & ((1 << FLAGS_log2_keys_per_lock) - 1)) == 0) {
+              range_locks.emplace_back(new MutexLock(
+                    shared->GetMutexForKey(rand_column_family, rand_key + j)));
+            }
+          }
+
+          keystr = Key(rand_key);
+          key = keystr;
+          column_family = column_families_[rand_column_family];
+          std::string end_keystr = Key(rand_key + FLAGS_range_deletion_width);
+          Slice end_key = end_keystr;
+          int covered = shared->DeleteRange(
+              rand_column_family, rand_key,
+              rand_key + FLAGS_range_deletion_width);
+          Status s = db_->DeleteRange(write_opts, column_family, key, end_key);
+          if (!s.ok()) {
+            fprintf(stderr, "delete range error: %s\n",
+                    s.ToString().c_str());
+            std::terminate();
+          }
+          thread->stats.AddRangeDeletions(1);
+          thread->stats.AddCoveredByRangeDeletions(covered);
+        }
       } else {
         // OPERATION iterate
         MultiIterate(thread, read_opts, column_family, key);
@@ -1987,6 +2067,7 @@ class StressTest {
     fprintf(stdout, "Prefix percentage         : %d%%\n", FLAGS_prefixpercent);
     fprintf(stdout, "Write percentage          : %d%%\n", FLAGS_writepercent);
     fprintf(stdout, "Delete percentage         : %d%%\n", FLAGS_delpercent);
+    fprintf(stdout, "Delete range percentage   : %d%%\n", FLAGS_delrangepercent);
     fprintf(stdout, "No overwrite percentage   : %d%%\n",
             FLAGS_nooverwritepercent);
     fprintf(stdout, "Iterate percentage        : %d%%\n", FLAGS_iterpercent);
@@ -2284,9 +2365,11 @@ int main(int argc, char** argv) {
     exit(1);
   }
   if ((FLAGS_readpercent + FLAGS_prefixpercent +
-       FLAGS_writepercent + FLAGS_delpercent + FLAGS_iterpercent) != 100) {
+       FLAGS_writepercent + FLAGS_delpercent + FLAGS_delrangepercent +
+       FLAGS_iterpercent) != 100) {
       fprintf(stderr,
-              "Error: Read+Prefix+Write+Delete+Iterate percents != 100!\n");
+              "Error: Read+Prefix+Write+Delete+DeleteRange+Iterate percents != "
+              "100!\n");
       exit(1);
   }
   if (FLAGS_disable_wal == 1 && FLAGS_reopen > 0) {
@@ -2301,6 +2384,11 @@ int main(int argc, char** argv) {
               (unsigned long)FLAGS_ops_per_thread);
       exit(1);
   }
+  if (FLAGS_test_batches_snapshots && FLAGS_delrangepercent > 0) {
+    fprintf(stderr, "Error: nonzero delrangepercent unsupported in "
+                    "test_batches_snapshots mode\n");
+    exit(1);
+  }
 
   // Choose a location for the test database if none given with --db=<path>
   if (FLAGS_db.empty()) {