bug-fix of ttl on compaction

db/compaction_job.cc

@@ -891,6 +891,9 @@ Status CompactionJob::Run() {
           output.meta.prop.inheritance_chain = tp->inheritance_chain;
           output.meta.prop.ratio_expire_time = tp->ratio_expire_time;
           output.meta.prop.scan_gap_expire_time = tp->scan_gap_expire_time;
+          // ROCKS_LOG_INFO(db_options_.info_log,
+          //                "ratio:%" PRIu64 ", scan:%" PRIu64,
+          //                tp->ratio_expire_time, tp->scan_gap_expire_time);
           output.finished = true;
           c->AddOutputTableFileNumber(file_number);
         }
@@ -2117,6 +2120,11 @@ Status CompactionJob::FinishCompactionOutputFile(
         tp.num_range_deletions > 0 ? 0 : TablePropertyCache::kNoRangeDeletions;
     meta->prop.flags |=
         tp.snapshots.empty() ? 0 : TablePropertyCache::kHasSnapshots;
+
+    meta->prop.ratio_expire_time = tp.ratio_expire_time;
+    meta->prop.scan_gap_expire_time = tp.scan_gap_expire_time;
+    ROCKS_LOG_INFO(db_options_.info_log, "ratio:%" PRIu64 ", scan:%" PRIu64,
+                   tp.ratio_expire_time, tp.scan_gap_expire_time);
   }
 
   if (s.ok() && tp.num_entries == 0 && tp.num_range_deletions == 0) {

db/db_impl.cc

@@ -17,6 +17,7 @@
 #endif
 
 #include <algorithm>
+#include <cinttypes>
 #include <cstdio>
 #include <map>
 #include <set>
@@ -929,12 +930,10 @@ void DBImpl::ScheduleGCTTL() {
   uint64_t marked_count = 0;
   uint64_t nowSeconds = env_->NowMicros() / 1000U / 1000U;
   auto should_marked_for_compacted = [&](uint64_t ratio_expire_time,
-                                        uint64_t scan_gap_expire_time,
-                                        uint64_t now) {
+                                         uint64_t scan_gap_expire_time,
+                                         uint64_t now) {
     ROCKS_LOG_INFO(immutable_db_options_.info_log,
-                   "SST Table property info: % PRIu64 "
-                   ",% PRIu64 "
-                   ",% PRIu64 ",
+                   "SST Table property info:%" PRIu64 ",%" PRIu64 ",%" PRIu64,
                    ratio_expire_time, scan_gap_expire_time, now);
     return (std::min(ratio_expire_time, scan_gap_expire_time) <= now);
   };

db/db_impl_gc_ttl_test.cc

@@ -24,6 +24,7 @@ class DBImplGCTTL_Test : public DBTestBase {
     options.ttl_scan_gap = 10;
     options.ttl_extractor_factory.reset(new test::TestTtlExtractorFactory());
     options.level0_file_num_compaction_trigger = 8;
+    options.enable_lazy_compaction = false;
     options.table_factory.reset(
         new BlockBasedTableFactory(BlockBasedTableOptions()));
   }
@@ -50,9 +51,7 @@ class DBImplGCTTL_Test : public DBTestBase {
                                                      flag = true;
                                                    });
     rocksdb::SyncPoint::GetInstance()->SetCallBack(
-        "DBImpl:ScheduleGCTTL-mark", [&](void* /*arg*/) {
-          mark++;
-        });
+        "DBImpl:ScheduleGCTTL-mark", [&](void* /*arg*/) { mark++; });
   }
 };
 
@@ -81,7 +80,7 @@ TEST_F(DBImplGCTTL_Test, L0FileExpiredTest) {
   dbfull()->TEST_WaitForStatsDumpRun([&] { mock_env_->set_current_time(ttl); });
   ASSERT_TRUE(flag);
   ASSERT_EQ(L0FilesNums, mark);
-  dbfull()->CompactRange(CompactRangeOptions(),nullptr, nullptr);
+  dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
   dbfull()->TEST_WaitForCompact();
   dbfull()->ScheduleGCTTL();
 

table/block_based_table_builder.cc

@@ -842,9 +842,9 @@ void BlockBasedTableBuilder::WritePropertiesBlock(
       uint64_t max_uint64_t = std::numeric_limits<uint64_t>::max();
 
       double ratio = rep_->moptions.ttl_garbage_collection_percentage;
+      uint64_t percentile_ratio_ttl = max_uint64_t;
       if (ratio <= 100.0) {
         assert(ttl_histogram_ != nullptr);
-        uint64_t percentile_ratio_ttl = max_uint64_t;
 
         if (!ttl_histogram_->Empty() &&
             kv_size_has_row_ttl_ >=
@@ -870,9 +870,8 @@ void BlockBasedTableBuilder::WritePropertiesBlock(
       ROCKS_LOG_INFO(rep_->ioptions.info_log,
                      "[%s] ratio_row_ttl:%" PRIu64 ", scan_gap_row_ttl:%" PRIu64
                      ".",
-                     rep_->column_family_name.c_str(),
-                     rep_->props.ratio_expire_time - now_seconds,
-                     rep_->props.scan_gap_expire_time - now_seconds);
+                     rep_->column_family_name.c_str(), percentile_ratio_ttl,
+                     min_ttl_seconds_);
       min_ttl_seconds_ = std::numeric_limits<uint64_t>::max();
       ttl_histogram_.reset();
       ttl_extractor_.reset();
@@ -935,8 +934,8 @@ Status BlockBasedTableBuilder::Finish(
     r->props.read_amp = prop->read_amp;
     r->props.dependence = prop->dependence;
     r->props.inheritance_chain = prop->inheritance_chain;
-    r->props.ratio_expire_time = prop->ratio_expire_time;
-    r->props.scan_gap_expire_time = prop->scan_gap_expire_time;
+    // r->props.ratio_expire_time = prop->ratio_expire_time;
+    // r->props.scan_gap_expire_time = prop->scan_gap_expire_time;
   }
   if (snapshots != nullptr) {
     r->props.snapshots = *snapshots;

table/meta_blocks.cc

@@ -10,6 +10,7 @@
 #include "db/table_properties_collector.h"
 #include "rocksdb/table.h"
 #include "rocksdb/table_properties.h"
+#include "rocksdb/terark_namespace.h"
 #include "table/block.h"
 #include "table/block_fetcher.h"
 #include "table/format.h"
@@ -19,7 +20,6 @@
 #include "util/coding.h"
 #include "util/file_reader_writer.h"
 
-#include "rocksdb/terark_namespace.h"
 namespace TERARKDB_NAMESPACE {
 
 MetaIndexBuilder::MetaIndexBuilder()
@@ -157,9 +157,11 @@ void PropertyBlockBuilder::AddTableProperty(const TableProperties& props) {
   if (!props.compression_name.empty()) {
     Add(TablePropertiesNames::kCompression, props.compression_name);
   }
-  if (true) {
+  if (props.ratio_expire_time < std::numeric_limits<uint64_t>::max()) {
     Add(TablePropertiesNames::kEarliestTimeBeginCompact,
         props.ratio_expire_time);
+  }
+  if (props.scan_gap_expire_time < std::numeric_limits<uint64_t>::max()) {
     Add(TablePropertiesNames::kLatestTimeEndCompact,
         props.scan_gap_expire_time);
   }
@@ -424,6 +426,20 @@ Status ReadProperties(const Slice& handle_value, RandomAccessFileReader* file,
       }
     } else if (key == TablePropertiesNames::kInheritanceChain) {
       GetUint64Vector(key, &raw_val, new_table_properties->inheritance_chain);
+    } else if (key == TablePropertiesNames::kRatioExpireTime) {
+      uint64_t u64_val;
+      if (!GetVarint64(&raw_val, &u64_val)) {
+        log_error();
+        continue;
+      }
+      new_table_properties->ratio_expire_time = u64_val;
+    } else if (key == TablePropertiesNames::kScanGapExpireTime) {
+      uint64_t u64_val;
+      if (!GetVarint64(&raw_val, &u64_val)) {
+        log_error();
+        continue;
+      }
+      new_table_properties->scan_gap_expire_time = u64_val;
     } else {
       // handle user-collected properties
       new_table_properties->user_collected_properties.insert(

table/table_properties.cc

@@ -181,17 +181,20 @@ void TableProperties::Add(const TableProperties& tp) {
   num_deletions += tp.num_deletions;
   num_merge_operands += tp.num_merge_operands;
   num_range_deletions += tp.num_range_deletions;
-  uint64_t max_uint64_t = std::numeric_limits<uint64_t>::max();
-  if (max_uint64_t - ratio_expire_time > tp.ratio_expire_time) {
-    ratio_expire_time += tp.ratio_expire_time;
-  } else {
-    ratio_expire_time = max_uint64_t;
-  }
-  if (max_uint64_t - scan_gap_expire_time > tp.scan_gap_expire_time) {
-    scan_gap_expire_time += tp.scan_gap_expire_time;
-  } else {
-    scan_gap_expire_time = max_uint64_t;
-  }
+  ratio_expire_time = std::min(ratio_expire_time, tp.ratio_expire_time);
+  scan_gap_expire_time =
+      std::min(scan_gap_expire_time, tp.scan_gap_expire_time);
+  // uint64_t max_uint64_t = std::numeric_limits<uint64_t>::max();
+  // if (max_uint64_t - ratio_expire_time > tp.ratio_expire_time) {
+  //   ratio_expire_time += tp.ratio_expire_time;
+  // } else {
+  //   ratio_expire_time = max_uint64_t;
+  // }
+  // if (max_uint64_t - scan_gap_expire_time > tp.scan_gap_expire_time) {
+  //   scan_gap_expire_time += tp.scan_gap_expire_time;
+  // } else {
+  //   scan_gap_expire_time = max_uint64_t;
+  // }
 }
 
 const std::string TablePropertiesNames::kDataSize = "rocksdb.data.size";