From 3c815464224f00f3f2f426382ea9ebbc9c957329 Mon Sep 17 00:00:00 2001
From: Igor Canadi <icanadi@fb.com>
Date: Wed, 26 Feb 2014 14:41:18 -0800
Subject: [PATCH] [CF] Make LogDeletionTest less flakey

Summary: Retry GetSortedWalFiles() and also wait 20ms before counting number of log files. WaitForFlush() doesn't necessarily wait for logs to be deleted, since logs are deleted outside of the mutex.

Test Plan: column_family_test

Reviewers: dhruba, haobo

CC: leveldb

Differential Revision: https://reviews.facebook.net/D16371
---
 db/column_family_test.cc | 28 +++++++++++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

diff --git a/db/column_family_test.cc b/db/column_family_test.cc
index 4e83a6e5d..208cb36b3 100644
--- a/db/column_family_test.cc
+++ b/db/column_family_test.cc
@@ -212,7 +212,21 @@ class ColumnFamilyTest {
   int CountLiveLogFiles() {
     int ret = 0;
     VectorLogPtr wal_files;
-    ASSERT_OK(db_->GetSortedWalFiles(wal_files));
+    Status s;
+    // GetSortedWalFiles is a flakey function -- it gets all the wal_dir
+    // children files and then later checks for their existance. if some of the
+    // log files doesn't exist anymore, it reports an error. it does all of this
+    // without DB mutex held, so if a background process deletes the log file
+    // while the function is being executed, it returns an error. We retry the
+    // function 10 times to avoid the error failing the test
+    for (int retries = 0; retries < 10; ++retries) {
+      wal_files.clear();
+      s = db_->GetSortedWalFiles(wal_files);
+      if (s.ok()) {
+        break;
+      }
+    }
+    ASSERT_OK(s);
     for (const auto& wal : wal_files) {
       if (wal->Type() == kAliveLogFile) {
         ++ret;
@@ -449,6 +463,7 @@ TEST(ColumnFamilyTest, FlushTest) {
 TEST(ColumnFamilyTest, LogDeletionTest) {
   column_family_options_.write_buffer_size = 100000;  // 100KB
   Open();
+  int micros_wait_for_log_deletion = 20000;
   CreateColumnFamilies({"one", "two", "three", "four"});
   // Each bracket is one log file. if number is in (), it means
   // we don't need it anymore (it's been flushed)
@@ -461,52 +476,63 @@ TEST(ColumnFamilyTest, LogDeletionTest) {
   PutRandomData(1, 1000, 100);
   WaitForFlush(1);
   // [0, (1)] [1]
+  env_->SleepForMicroseconds(micros_wait_for_log_deletion);
   ASSERT_EQ(CountLiveLogFiles(), 2);
   PutRandomData(0, 1, 100);
   // [0, (1)] [0, 1]
+  env_->SleepForMicroseconds(micros_wait_for_log_deletion);
   ASSERT_EQ(CountLiveLogFiles(), 2);
   PutRandomData(2, 1, 100);
   // [0, (1)] [0, 1, 2]
   PutRandomData(2, 1000, 100);
   WaitForFlush(2);
   // [0, (1)] [0, 1, (2)] [2]
+  env_->SleepForMicroseconds(micros_wait_for_log_deletion);
   ASSERT_EQ(CountLiveLogFiles(), 3);
   PutRandomData(2, 1000, 100);
   WaitForFlush(2);
   // [0, (1)] [0, 1, (2)] [(2)] [2]
+  env_->SleepForMicroseconds(micros_wait_for_log_deletion);
   ASSERT_EQ(CountLiveLogFiles(), 4);
   PutRandomData(3, 1, 100);
   // [0, (1)] [0, 1, (2)] [(2)] [2, 3]
   PutRandomData(1, 1, 100);
   // [0, (1)] [0, 1, (2)] [(2)] [1, 2, 3]
+  env_->SleepForMicroseconds(micros_wait_for_log_deletion);
   ASSERT_EQ(CountLiveLogFiles(), 4);
   PutRandomData(1, 1000, 100);
   WaitForFlush(1);
   // [0, (1)] [0, (1), (2)] [(2)] [(1), 2, 3] [1]
+  env_->SleepForMicroseconds(micros_wait_for_log_deletion);
   ASSERT_EQ(CountLiveLogFiles(), 5);
   PutRandomData(0, 1000, 100);
   WaitForFlush(0);
   // [(0), (1)] [(0), (1), (2)] [(2)] [(1), 2, 3] [1, (0)] [0]
   // delete obsolete logs -->
   // [(1), 2, 3] [1, (0)] [0]
+  env_->SleepForMicroseconds(micros_wait_for_log_deletion);
   ASSERT_EQ(CountLiveLogFiles(), 3);
   PutRandomData(0, 1000, 100);
   WaitForFlush(0);
   // [(1), 2, 3] [1, (0)], [(0)] [0]
+  env_->SleepForMicroseconds(micros_wait_for_log_deletion);
   ASSERT_EQ(CountLiveLogFiles(), 4);
   PutRandomData(1, 1000, 100);
   WaitForFlush(1);
   // [(1), 2, 3] [(1), (0)] [(0)] [0, (1)] [1]
+  env_->SleepForMicroseconds(micros_wait_for_log_deletion);
   ASSERT_EQ(CountLiveLogFiles(), 5);
   PutRandomData(2, 1000, 100);
   WaitForFlush(2);
   // [(1), (2), 3] [(1), (0)] [(0)] [0, (1)] [1, (2)], [2]
+  env_->SleepForMicroseconds(micros_wait_for_log_deletion);
   ASSERT_EQ(CountLiveLogFiles(), 6);
   PutRandomData(3, 1000, 100);
   WaitForFlush(3);
   // [(1), (2), (3)] [(1), (0)] [(0)] [0, (1)] [1, (2)], [2, (3)] [3]
   // delete obsolete logs -->
   // [0, (1)] [1, (2)], [2, (3)] [3]
+  env_->SleepForMicroseconds(micros_wait_for_log_deletion);
   ASSERT_EQ(CountLiveLogFiles(), 4);
   Close();
 }
-- 
GitLab