[WIP] ...

db/c.cc

@@ -2551,6 +2551,14 @@ void rocksdb_options_set_enable_lazy_compaction(rocksdb_options_t* opt, int enab
   opt->rep.enable_lazy_compaction = enable;
 }
 
+void rocksdb_options_set_blob_size(rocksdb_options_t* opt, size_t blob_size) {
+  opt->rep.blob_size = blob_size;
+}
+
+void rocksdb_options_set_blob_gc_ratio(rocksdb_options_t* opt, int ratio) {
+  opt->rep.blob_gc_ratio = ratio;
+}
+
 void rocksdb_options_set_optimize_filters_for_hits(rocksdb_options_t* opt, int v) {
   opt->rep.optimize_filters_for_hits = v;
 }

db/version_builder.cc

@@ -116,6 +116,7 @@ class VersionBuilder::Rep {
   LevelState* levels_;
   std::unordered_map<uint64_t, size_t> depend_map_;
   std::vector<FileMetaData*> depend_files_;
+  std::unordered_map<uint64_t, uint64_t> update_antiquation_;
   // Store states of levels larger than num_levels_. We do this instead of
   // storing them in levels_ to avoid regression in case there are no files
   // on invalid levels. The version is not consistent if in the end the files
@@ -408,6 +409,10 @@ class VersionBuilder::Rep {
       UnrefFile(depend_files_[i]);
     }
     depend_files_.resize(depend_file_count);
+
+    for (auto& pair : edit->GetAntiquation()) {
+      update_antiquation_[pair.first] += pair.second;
+    }
   }
 
   // Save the current state in *v.
@@ -456,6 +461,10 @@ class VersionBuilder::Rep {
 #endif
 
       auto maybe_add_file = [&](FileMetaData* f) {
+        auto find = update_antiquation_.find(f->fd.GetNumber());
+        if (find != update_antiquation_.end()) {
+          f->num_antiquation += find->second;
+        }
         if (levels_[level].deleted_files.count(f->fd.GetNumber()) > 0) {
           deleted_files.push_back(f);
         } else {

db/version_edit.cc

@@ -124,8 +124,9 @@ bool VersionEdit::EncodeTo(std::string* dst) const {
       return false;
     }
     bool has_customized_fields = false;
-    if (f.marked_for_compaction || has_min_log_number_to_keep_ ||
-        f.sst_purpose != 0) {
+    if (f.num_antiquation > 0 || f.marked_for_compaction ||
+        has_min_log_number_to_keep_ || f.sst_purpose != 0 ||
+        !f.sst_depend.empty()) {
       PutVarint32(dst, kNewFile4);
       has_customized_fields = true;
     } else if (f.fd.GetPathId() == 0) {
@@ -176,6 +177,9 @@ bool VersionEdit::EncodeTo(std::string* dst) const {
         char p = static_cast<char>(f.fd.GetPathId());
         PutLengthPrefixedSlice(dst, Slice(&p, 1));
       }
+      if (f.num_antiquation > 0) {
+        PutVarint32Varint64(dst, CustomTag::kNumAntiquation, f.num_antiquation);
+      }
       if (f.marked_for_compaction) {
         PutVarint32(dst, CustomTag::kNeedCompaction);
         char p = static_cast<char>(1);
@@ -293,9 +297,13 @@ const char* VersionEdit::DecodeNewFile4From(Slice* input) {
           }
           path_id = field[0];
           if (path_id > 3) {
-            return "path_id wrong vaue";
+            return "path_id wrong value";
           }
           break;
+        case kNumAntiquation:
+          if (!GetVarint64(&field, &f.num_antiquation)) {
+            return "num_antiquation field";
+          }
         case kNeedCompaction:
           if (field.size() != 1) {
             return "need_compaction field wrong size";
@@ -517,8 +525,8 @@ Status VersionEdit::DecodeFrom(const Slice& src) {
         if (!GetVarint64(&input, &update_antiquation_size)) {
           if (!msg) {
             msg = update_antiquation_msg;
-            break;
           }
+          break;
         }
         update_antiquation_.resize(update_antiquation_size);
         for (auto& pair : update_antiquation_) {

include/rocksdb/options.h

@@ -286,6 +286,12 @@ struct ColumnFamilyOptions : public AdvancedColumnFamilyOptions {
   // Enable map or link compaction
   bool enable_lazy_compaction = false;
 
+  //
+  size_t blob_size = 1024;
+
+  //
+  double blob_gc_ratio = 0.05;
+
   // This is a factory that provides TableFactory objects.
   // Default: a block-based table factory that provides a default
   // implementation of TableBuilder and TableReader with default

options/cf_options.cc

@@ -147,6 +147,10 @@ void MutableCFOptions::Dump(Logger* log) const {
                  disable_auto_compactions);
   ROCKS_LOG_INFO(log, "                   enable_lazy_compaction: %d",
                  enable_lazy_compaction);
+  ROCKS_LOG_INFO(log, "                                blob_size: %d",
+                 blob_size);
+  ROCKS_LOG_INFO(log, "                            blob_gc_ratio: %d",
+                 blob_gc_ratio);
   ROCKS_LOG_INFO(log, "      soft_pending_compaction_bytes_limit: %" PRIu64,
                  soft_pending_compaction_bytes_limit);
   ROCKS_LOG_INFO(log, "      hard_pending_compaction_bytes_limit: %" PRIu64,

options/cf_options.h

@@ -138,6 +138,8 @@ struct MutableCFOptions {
         prefix_extractor(options.prefix_extractor),
         disable_auto_compactions(options.disable_auto_compactions),
         enable_lazy_compaction(options.enable_lazy_compaction),
+        blob_size(options.blob_size),
+        blob_gc_ratio(options.blob_gc_ratio),
         soft_pending_compaction_bytes_limit(
             options.soft_pending_compaction_bytes_limit),
         hard_pending_compaction_bytes_limit(
@@ -175,6 +177,8 @@ struct MutableCFOptions {
         prefix_extractor(nullptr),
         disable_auto_compactions(false),
         enable_lazy_compaction(false),
+        blob_size(0),
+        blob_gc_ratio(0),
         soft_pending_compaction_bytes_limit(0),
         hard_pending_compaction_bytes_limit(0),
         level0_file_num_compaction_trigger(0),
@@ -225,6 +229,8 @@ struct MutableCFOptions {
   // Compaction related options
   bool disable_auto_compactions;
   bool enable_lazy_compaction;
+  size_t blob_size;
+  double blob_gc_ratio;
   uint64_t soft_pending_compaction_bytes_limit;
   uint64_t hard_pending_compaction_bytes_limit;
   int level0_file_num_compaction_trigger;

options/options_helper.cc

@@ -153,6 +153,8 @@ ColumnFamilyOptions BuildColumnFamilyOptions(
       mutable_cf_options.disable_auto_compactions;
   cf_opts.enable_lazy_compaction =
       mutable_cf_options.enable_lazy_compaction;
+  cf_opts.blob_size = mutable_cf_options.blob_size;
+  cf_opts.blob_gc_ratio = mutable_cf_options.blob_gc_ratio;
   cf_opts.soft_pending_compaction_bytes_limit =
       mutable_cf_options.soft_pending_compaction_bytes_limit;
   cf_opts.hard_pending_compaction_bytes_limit =
@@ -1689,6 +1691,14 @@ std::unordered_map<std::string, OptionTypeInfo>
          {offset_of(&ColumnFamilyOptions::enable_lazy_compaction),
           OptionType::kBoolean, OptionVerificationType::kNormal, true,
           offsetof(struct MutableCFOptions, enable_lazy_compaction)}},
+        {"blob_size",
+         {offset_of(&ColumnFamilyOptions::blob_size),
+          OptionType::kSizeT, OptionVerificationType::kNormal, true,
+          offsetof(struct MutableCFOptions, blob_size)}},
+        {"enable_lazy_compaction",
+         {offset_of(&ColumnFamilyOptions::blob_gc_ratio),
+          OptionType::kDouble, OptionVerificationType::kNormal, true,
+          offsetof(struct MutableCFOptions, blob_gc_ratio)}},
         {"filter_deletes",
          {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, true,
           0}},

options/options_settable_test.cc

@@ -446,6 +446,8 @@ TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) {
       "hard_pending_compaction_bytes_limit=0;"
       "disable_auto_compactions=false;"
       "enable_lazy_compaction=false;"
+      "blob_size=1024;"
+      "blob_gc_ratio=0.05;"
       "report_bg_io_stats=true;"
       "ttl=60;"
       "compaction_options_fifo={max_table_files_size=3;ttl=100;allow_"

tools/db_bench_tool.cc

@@ -923,6 +923,10 @@ DEFINE_bool(disable_auto_compactions, false, "Do not auto trigger compactions");
 
 DEFINE_bool(enable_lazy_compaction, false, "Enable map or link compaction");
 
+DEFINE_uint64(blob_size, 1024, "Key Value Separate blob size");
+
+DEFINE_double(blob_gc_ratio, 0.05, "Blob SST gc ratio");
+
 DEFINE_uint64(wal_ttl_seconds, 0, "Set the TTL for the WAL Files in seconds.");
 DEFINE_uint64(wal_size_limit_MB, 0, "Set the size limit for the WAL Files"
               " in MB.");
@@ -3408,6 +3412,8 @@ void VerifyDBFromDB(std::string& truth_db_name) {
     options.max_compaction_bytes = FLAGS_max_compaction_bytes;
     options.disable_auto_compactions = FLAGS_disable_auto_compactions;
     options.enable_lazy_compaction = FLAGS_enable_lazy_compaction;
+    options.blob_size = FLAGS_blob_size;
+    options.blob_gc_tario = FLOAGS_blob_gc_ratio;
     options.optimize_filters_for_hits = FLAGS_optimize_filters_for_hits;
 
     // fill storage options