Several easy-to-add properties related to compaction and flushes

Summary: To partly address the request @nkg- raised, add three easy-to-add properties to compactions and flushes.

Test Plan: run unit tests and add a new unit test to cover new properties.

Reviewers: haobo, dhruba

Reviewed By: dhruba

CC: nkg-, leveldb

Differential Revision: https://reviews.facebook.net/D13677

db/db_impl.cc

@@ -3696,9 +3696,10 @@ const Options& DBImpl::GetOptions() const {
 
 bool DBImpl::GetProperty(const Slice& property, std::string* value) {
   value->clear();
+  DBPropertyType property_type = GetPropertyType(property);
   MutexLock l(&mutex_);
-  return internal_stats_.GetProperty(property, value, versions_.get(),
-                                     imm_.size());
+  return internal_stats_.GetProperty(property_type, property, value,
+                                     versions_.get(), imm_);
 }
 
 void DBImpl::GetApproximateSizes(

db/db_test.cc

@@ -2055,6 +2055,94 @@ TEST(DBTest, NumImmutableMemTable) {
   } while (ChangeCompactOptions());
 }
 
+class SleepingBackgroundTask {
+ public:
+  explicit SleepingBackgroundTask(Env* env)
+      : env_(env), bg_cv_(&mutex_), should_sleep_(true) {}
+  void DoSleep() {
+    MutexLock l(&mutex_);
+    while (should_sleep_) {
+      bg_cv_.Wait();
+    }
+  }
+  void WakeUp() {
+    MutexLock l(&mutex_);
+    should_sleep_ = false;
+    bg_cv_.SignalAll();
+  }
+
+  static void DoSleepTask(void* arg) {
+    reinterpret_cast<SleepingBackgroundTask*>(arg)->DoSleep();
+  }
+
+ private:
+  const Env* env_;
+  port::Mutex mutex_;
+  port::CondVar bg_cv_;  // Signalled when background work finishes
+  bool should_sleep_;
+};
+
+TEST(DBTest, GetProperty) {
+  // Set sizes to both background thread pool to be 1 and block them.
+  env_->SetBackgroundThreads(1, Env::HIGH);
+  env_->SetBackgroundThreads(1, Env::LOW);
+  SleepingBackgroundTask sleeping_task_low(env_);
+  env_->Schedule(&SleepingBackgroundTask::DoSleepTask, &sleeping_task_low,
+                 Env::Priority::LOW);
+  SleepingBackgroundTask sleeping_task_high(env_);
+  env_->Schedule(&SleepingBackgroundTask::DoSleepTask, &sleeping_task_high,
+                 Env::Priority::HIGH);
+
+  Options options = CurrentOptions();
+  WriteOptions writeOpt = WriteOptions();
+  writeOpt.disableWAL = true;
+  options.compaction_style = kCompactionStyleUniversal;
+  options.level0_file_num_compaction_trigger = 1;
+  options.compaction_options_universal.size_ratio = 50;
+  options.max_background_compactions = 1;
+  options.max_background_flushes = 1;
+  options.max_write_buffer_number = 10;
+  options.min_write_buffer_number_to_merge = 1;
+  options.write_buffer_size = 1000000;
+  Reopen(&options);
+
+  std::string big_value(1000000 * 2, 'x');
+  std::string num;
+  SetPerfLevel(kEnableTime);
+
+  ASSERT_OK(dbfull()->Put(writeOpt, "k1", big_value));
+  ASSERT_TRUE(dbfull()->GetProperty("rocksdb.num-immutable-mem-table", &num));
+  ASSERT_EQ(num, "0");
+  ASSERT_TRUE(dbfull()->GetProperty("rocksdb.mem-table-flush-pending", &num));
+  ASSERT_EQ(num, "0");
+  ASSERT_TRUE(dbfull()->GetProperty("rocksdb.compaction-pending", &num));
+  ASSERT_EQ(num, "0");
+  perf_context.Reset();
+
+  ASSERT_OK(dbfull()->Put(writeOpt, "k2", big_value));
+  ASSERT_TRUE(dbfull()->GetProperty("rocksdb.num-immutable-mem-table", &num));
+  ASSERT_EQ(num, "1");
+  ASSERT_OK(dbfull()->Put(writeOpt, "k3", big_value));
+  ASSERT_TRUE(dbfull()->GetProperty("rocksdb.num-immutable-mem-table", &num));
+  ASSERT_EQ(num, "2");
+  ASSERT_TRUE(dbfull()->GetProperty("rocksdb.mem-table-flush-pending", &num));
+  ASSERT_EQ(num, "1");
+  ASSERT_TRUE(dbfull()->GetProperty("rocksdb.compaction-pending", &num));
+  ASSERT_EQ(num, "0");
+
+  sleeping_task_high.WakeUp();
+  dbfull()->TEST_WaitForFlushMemTable();
+
+  ASSERT_OK(dbfull()->Put(writeOpt, "k4", big_value));
+  ASSERT_OK(dbfull()->Put(writeOpt, "k5", big_value));
+  dbfull()->TEST_WaitForFlushMemTable();
+  ASSERT_TRUE(dbfull()->GetProperty("rocksdb.mem-table-flush-pending", &num));
+  ASSERT_EQ(num, "0");
+  ASSERT_TRUE(dbfull()->GetProperty("rocksdb.compaction-pending", &num));
+  ASSERT_EQ(num, "1");
+  sleeping_task_low.WakeUp();
+}
+
 TEST(DBTest, FLUSH) {
   do {
     Options options = CurrentOptions();

db/internal_stats.h

@@ -17,6 +17,23 @@
 #include <string>
 
 namespace rocksdb {
+
+class MemTableList;
+
+enum DBPropertyType {
+  kNumFilesAtLevel,  // Number of files at a specific level
+  kLevelStats,       // Return number of files and total sizes of each level
+  kStats,            // Return general statitistics of DB
+  kSsTables,         // Return a human readable string of current SST files
+  kNumImmutableMemTable,  // Return number of immutable mem tables
+  MemtableFlushPending,  // Return 1 if mem table flushing is pending, otherwise
+                         // 0.
+  CompactionPending,     // Return 1 if a compaction is pending. Otherwise 0.
+  kUnknown,
+};
+
+extern DBPropertyType GetPropertyType(const Slice& property);
+
 class InternalStats {
  public:
   enum WriteStallType {
@@ -99,8 +116,9 @@ class InternalStats {
     stall_leveln_slowdown_count_[level] += micros;
   }
 
-  bool GetProperty(const Slice& property, std::string* value,
-                   VersionSet* version_set, int immsize);
+  bool GetProperty(DBPropertyType property_type, const Slice& property,
+                   std::string* value, VersionSet* version_set,
+                   const MemTableList& imm);
 
  private:
   std::vector<CompactionStats> compaction_stats_;

db/memtable_list.cc

@@ -92,7 +92,7 @@ void MemTableListVersion::Remove(MemTable* m) {
 
 // Returns true if there is at least one memtable on which flush has
 // not yet started.
-bool MemTableList::IsFlushPending() {
+bool MemTableList::IsFlushPending() const {
   if ((flush_requested_ && num_flush_not_started_ >= 1) ||
       (num_flush_not_started_ >= min_write_buffer_number_to_merge_)) {
     assert(imm_flush_needed.NoBarrier_Load() != nullptr);

db/memtable_list.h

@@ -86,7 +86,7 @@ class MemTableList {
 
   // Returns true if there is at least one memtable on which flush has
   // not yet started.
-  bool IsFlushPending();
+  bool IsFlushPending() const;
 
   // Returns the earliest memtables that needs to be flushed. The returned
   // memtables are guaranteed to be in the ascending order of created time.