merge ljc

db/db_impl.cc

@@ -100,8 +100,6 @@
 #if !defined(_MSC_VER) && !defined(__APPLE__)
 #include <sys/unistd.h>
 
-
-
 #endif
 #include "utilities/util/valvec.hpp"
 
@@ -112,8 +110,9 @@
 
 #ifdef WITH_TERARK_ZIP
 #include <table/terark_zip_table.h>
-#include <terark/util/fiber_pool.hpp>
+
 #include <terark/thread/fiber_yield.hpp>
+#include <terark/util/fiber_pool.hpp>
 #endif
 
 #ifdef BOOSTLIB
@@ -121,7 +120,6 @@
 #endif
 //#include <boost/context/pooled_fixedsize_stack.hpp>
 
-
 #ifdef __GNUC__
 #pragma GCC diagnostic pop
 #endif
@@ -773,8 +771,41 @@ void DBImpl::StartTimedTasks() {
     }
   }
 }
-
+void DBImpl::ScheduleGCTTL() {
+  uint64_t mark_count = 0;
+  uint64_t marked_count = 0;
+  uint64_t nowSeconds = env_->NowMicros() / 1000000;
+  auto should_marked_for_compacted = [](uint64_t ratio_expire_time,
+                                        uint64_t scan_gap_expire_time,
+                                        uint64_t now) {
+    return (std::min(ratio_expire_time, scan_gap_expire_time) < now);
+  };
+  ROCKS_LOG_INFO(immutable_db_options_.info_log, "Start ScheduleGCTTL");
+  for (auto cfd : *versions_->GetColumnFamilySet()) {
+    if (cfd->initialized()) {
+      VersionStorageInfo* vsi = cfd->current()->storage_info();
+      for (int l = 0; l < vsi->num_levels(); l++) {
+        for (auto sst : vsi->LevelFiles(l)) {
+          if (sst->marked_for_compaction) marked_count++;
+          if (!sst->marked_for_compaction)
+            sst->marked_for_compaction = should_marked_for_compacted(
+                sst->prop.ratio_expire_time, sst->prop.scan_gap_expire_time,
+                nowSeconds);
+          if (sst->marked_for_compaction) {
+            mark_count++;
+          }
+        }
+      }
+    }
+  }
+  ROCKS_LOG_INFO(immutable_db_options_.info_log, "marked for compact SST: %d,%d",
+                 marked_count,mark_count);
+  if (mark_count > 0) {
+    MaybeScheduleFlushOrCompaction();
+  }
+}
 void DBImpl::DumpStats() {
+  ScheduleGCTTL();
   TEST_SYNC_POINT("DBImpl::DumpStats:1");
 #ifndef ROCKSDB_LITE
   const DBPropertyInfo* cf_property_info =
@@ -1259,9 +1290,9 @@ void DBImpl::SchedulePurge() {
 void DBImpl::BackgroundCallPurge() {
   mutex_.Lock();
 
-  // We use one single loop to clear both queues so that after existing the loop
-  // both queues are empty. This is stricter than what is needed, but can make
-  // it easier for us to reason the correctness.
+  // We use one single loop to clear both queues so that after existing the
+  // loop both queues are empty. This is stricter than what is needed, but can
+  // make it easier for us to reason the correctness.
   while (!purge_queue_.empty() | !superversion_to_free_queue_.empty() |
          !logs_to_free_queue_.empty()) {
     if (!superversion_to_free_queue_.empty()) {
@@ -2249,8 +2280,8 @@ Status DBImpl::NewIterators(
 #endif
   } else {
     // Note: no need to consider the special case of
-    // last_seq_same_as_publish_seq_==false since NewIterators is overridden in
-    // WritePreparedTxnDB
+    // last_seq_same_as_publish_seq_==false since NewIterators is overridden
+    // in WritePreparedTxnDB
     auto snapshot = read_options.snapshot != nullptr
                         ? read_options.snapshot->GetSequenceNumber()
                         : versions_->LastSequence();

db/db_impl.h

@@ -1151,6 +1151,9 @@ class DBImpl : public DB {
   // dump rocksdb.stats to LOG
   void DumpStats();
 
+  //
+  void ScheduleGCTTL();
+
   // Return the minimum empty level that could hold the total data in the
   // input level. Return the input level, if such level could not be found.
   int FindMinimumEmptyLevelFitting(ColumnFamilyData* cfd,

db/db_impl_gc_ttl_test.cc

+// Copyright (c) 2020-present, Bytedance Inc.  All rights reserved.
+// This source code is licensed under Apache 2.0 License.
+
+#include "table/block_based_table_builder.h"
+
+#include "include/rocksdb/ttl_extractor.h"
+#include "util/testharness.h"
+#include "util/testutil.h"
+
+namespace rocksdb{
+
+class DBImplGCTTL_Test : public testing::Test {
+  //
+
+ private:
+
+};
+
+TEST_F(DBImplGCTTL_Test,OpenTest) {
+
+}
+}
+int main(int argc, char** argv) {
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
\ No newline at end of file