feat(metrics): Add bloom filter related metrics (#521)

rdsn @ a0ecf81b

-Subproject commit 3b1133b269c26f52f6c9d8718d1b904fcb440b0f
+Subproject commit a0ecf81b655f781dff3271d68632cdca170bd323

src/server/info_collector.cpp

@@ -209,6 +209,9 @@ info_collector::app_stat_counters *info_collector::get_app_counters(const std::s
     INIT_COUNTER(rdb_index_and_filter_blocks_mem_usage);
     INIT_COUNTER(rdb_memtable_mem_usage);
     INIT_COUNTER(rdb_estimate_num_keys);
+    INIT_COUNTER(rdb_bf_seek_negatives_rate);
+    INIT_COUNTER(rdb_bf_point_negatives_rate);
+    INIT_COUNTER(rdb_bf_point_false_positive_rate);
     INIT_COUNTER(read_qps);
     INIT_COUNTER(write_qps);
     INIT_COUNTER(backup_request_qps);

src/server/info_collector.h

@@ -31,6 +31,11 @@ class info_collector
 public:
     struct app_stat_counters
     {
+        double convert_to_1M_ratio(double hit, double total)
+        {
+            return std::abs(total) < 1e-6 ? 0 : hit / total * 1e6;
+        }
+
         void set(const table_stats &row_stats)
         {
             get_qps->set(row_stats.total_get_qps);
@@ -55,14 +60,24 @@ public:
             storage_mb->set(row_stats.total_storage_mb);
             storage_count->set(row_stats.total_storage_count);
             rdb_block_cache_hit_rate->set(
-                std::abs(row_stats.total_rdb_block_cache_total_count) < 1e-6
-                    ? 0
-                    : row_stats.total_rdb_block_cache_hit_count /
-                          row_stats.total_rdb_block_cache_total_count * 1000000);
+                convert_to_1M_ratio(row_stats.total_rdb_block_cache_hit_count,
+                                    row_stats.total_rdb_block_cache_total_count));
             rdb_index_and_filter_blocks_mem_usage->set(
                 row_stats.total_rdb_index_and_filter_blocks_mem_usage);
             rdb_memtable_mem_usage->set(row_stats.total_rdb_memtable_mem_usage);
             rdb_estimate_num_keys->set(row_stats.total_rdb_estimate_num_keys);
+            rdb_bf_seek_negatives_rate->set(convert_to_1M_ratio(
+                row_stats.total_rdb_bf_seek_negatives, row_stats.total_rdb_bf_seek_total));
+            rdb_bf_point_negatives_rate->set(
+                convert_to_1M_ratio(row_stats.total_rdb_bf_point_negatives,
+                                    row_stats.total_rdb_bf_point_negatives +
+                                        row_stats.total_rdb_bf_point_positive_total));
+            rdb_bf_point_false_positive_rate->set(
+                convert_to_1M_ratio(row_stats.total_rdb_bf_point_positive_total -
+                                        row_stats.total_rdb_bf_point_positive_true,
+                                    (row_stats.total_rdb_bf_point_positive_total -
+                                     row_stats.total_rdb_bf_point_positive_true) +
+                                        row_stats.total_rdb_bf_point_negatives));
             read_qps->set(row_stats.get_total_read_qps());
             write_qps->set(row_stats.get_total_write_qps());
             backup_request_qps->set(row_stats.total_backup_request_qps);
@@ -101,6 +116,9 @@ public:
         ::dsn::perf_counter_wrapper rdb_index_and_filter_blocks_mem_usage;
         ::dsn::perf_counter_wrapper rdb_memtable_mem_usage;
         ::dsn::perf_counter_wrapper rdb_estimate_num_keys;
+        ::dsn::perf_counter_wrapper rdb_bf_seek_negatives_rate;
+        ::dsn::perf_counter_wrapper rdb_bf_point_negatives_rate;
+        ::dsn::perf_counter_wrapper rdb_bf_point_false_positive_rate;
         ::dsn::perf_counter_wrapper read_qps;
         ::dsn::perf_counter_wrapper write_qps;
         ::dsn::perf_counter_wrapper backup_request_qps;

src/server/pegasus_server_impl.cpp

@@ -431,6 +431,44 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r)
         name,
         COUNTER_TYPE_NUMBER,
         "statistics the estimated number of keys inside the rocksdb");
+
+    snprintf(name, 255, "rdb.bf_seek_negatives@%s", str_gpid.c_str());
+    _pfc_rdb_bf_seek_negatives.init_app_counter("app.pegasus",
+                                                name,
+                                                COUNTER_TYPE_NUMBER,
+                                                "statistics the number of times bloom filter was "
+                                                "checked before creating iterator on a file and "
+                                                "useful in avoiding iterator creation (and thus "
+                                                "likely IOPs)");
+
+    snprintf(name, 255, "rdb.bf_seek_total@%s", str_gpid.c_str());
+    _pfc_rdb_bf_seek_total.init_app_counter("app.pegasus",
+                                            name,
+                                            COUNTER_TYPE_NUMBER,
+                                            "statistics the number of times bloom filter was "
+                                            "checked before creating iterator on a file");
+
+    snprintf(name, 255, "rdb.bf_point_positive_true@%s", str_gpid.c_str());
+    _pfc_rdb_bf_point_positive_true.init_app_counter(
+        "app.pegasus",
+        name,
+        COUNTER_TYPE_NUMBER,
+        "statistics the number of times bloom filter has avoided file reads, i.e., negatives");
+
+    snprintf(name, 255, "rdb.bf_point_positive_total@%s", str_gpid.c_str());
+    _pfc_rdb_bf_point_positive_total.init_app_counter(
+        "app.pegasus",
+        name,
+        COUNTER_TYPE_NUMBER,
+        "statistics the number of times bloom FullFilter has not avoided the reads");
+
+    snprintf(name, 255, "rdb.bf_point_negatives@%s", str_gpid.c_str());
+    _pfc_rdb_bf_point_negatives.init_app_counter("app.pegasus",
+                                                 name,
+                                                 COUNTER_TYPE_NUMBER,
+                                                 "statistics the number of times bloom FullFilter "
+                                                 "has not avoided the reads and data actually "
+                                                 "exist");
 }
 
 void pegasus_server_impl::parse_checkpoints()
@@ -2408,6 +2446,8 @@ void pegasus_server_impl::update_replica_rocksdb_statistics()
 {
     std::string str_val;
     uint64_t val = 0;
+
+    // Update _pfc_rdb_sst_count
     for (int i = 0; i < _data_cf_opts.num_levels; ++i) {
         int cur_level_count = 0;
         if (_db->GetProperty(rocksdb::DB::Properties::kNumFilesAtLevelPrefix + std::to_string(i),
@@ -2419,6 +2459,7 @@ void pegasus_server_impl::update_replica_rocksdb_statistics()
     _pfc_rdb_sst_count->set(val);
     dinfo_replica("_pfc_rdb_sst_count: {}", val);
 
+    // Update _pfc_rdb_sst_size
     if (_db->GetProperty(_data_cf, rocksdb::DB::Properties::kTotalSstFilesSize, &str_val) &&
         dsn::buf2uint64(str_val, val)) {
         static uint64_t bytes_per_mb = 1U << 20U;
@@ -2426,6 +2467,7 @@ void pegasus_server_impl::update_replica_rocksdb_statistics()
         dinfo_replica("_pfc_rdb_sst_size: {} bytes", val);
     }
 
+    // Update _pfc_rdb_block_cache_hit_count and _pfc_rdb_block_cache_total_count
     uint64_t block_cache_hit = _statistics->getTickerCount(rocksdb::BLOCK_CACHE_HIT);
     _pfc_rdb_block_cache_hit_count->set(block_cache_hit);
     dinfo_replica("_pfc_rdb_block_cache_hit_count: {}", block_cache_hit);
@@ -2435,29 +2477,60 @@ void pegasus_server_impl::update_replica_rocksdb_statistics()
     _pfc_rdb_block_cache_total_count->set(block_cache_total);
     dinfo_replica("_pfc_rdb_block_cache_total_count: {}", block_cache_total);
 
+    // Update _pfc_rdb_index_and_filter_blocks_mem_usage
     if (_db->GetProperty(_data_cf, rocksdb::DB::Properties::kEstimateTableReadersMem, &str_val) &&
         dsn::buf2uint64(str_val, val)) {
         _pfc_rdb_index_and_filter_blocks_mem_usage->set(val);
         dinfo_replica("_pfc_rdb_index_and_filter_blocks_mem_usage: {} bytes", val);
     }
 
+    // Update _pfc_rdb_memtable_mem_usage
     if (_db->GetProperty(_data_cf, rocksdb::DB::Properties::kCurSizeAllMemTables, &str_val) &&
         dsn::buf2uint64(str_val, val)) {
         _pfc_rdb_memtable_mem_usage->set(val);
         dinfo_replica("_pfc_rdb_memtable_mem_usage: {} bytes", val);
     }
 
-    // for the same n kv pairs, kEstimateNumKeys will be counted n times, you need compaction to
-    // remove duplicate
+    // Update _pfc_rdb_estimate_num_keys
+    // NOTE: for the same n kv pairs, kEstimateNumKeys will be counted n times, you need compaction
+    // to remove duplicate
     if (_db->GetProperty(_data_cf, rocksdb::DB::Properties::kEstimateNumKeys, &str_val) &&
         dsn::buf2uint64(str_val, val)) {
         _pfc_rdb_estimate_num_keys->set(val);
         dinfo_replica("_pfc_rdb_estimate_num_keys: {}", val);
     }
+
+    // Update _pfc_rdb_bf_seek_negatives
+    uint64_t bf_seek_negatives = _statistics->getTickerCount(rocksdb::BLOOM_FILTER_PREFIX_USEFUL);
+    _pfc_rdb_bf_seek_negatives->set(bf_seek_negatives);
+    dinfo_replica("_pfc_rdb_bf_seek_negatives: {}", bf_seek_negatives);
+
+    // Update _pfc_rdb_bf_seek_total
+    uint64_t bf_seek_total = _statistics->getTickerCount(rocksdb::BLOOM_FILTER_PREFIX_CHECKED);
+    _pfc_rdb_bf_seek_total->set(bf_seek_total);
+    dinfo_replica("_pfc_rdb_bf_seek_total: {}", bf_seek_total);
+
+    // Update _pfc_rdb_bf_point_positive_true
+    uint64_t bf_point_positive_true =
+        _statistics->getTickerCount(rocksdb::BLOOM_FILTER_FULL_TRUE_POSITIVE);
+    _pfc_rdb_bf_point_positive_true->set(bf_point_positive_true);
+    dinfo_replica("_pfc_rdb_bf_point_positive_true: {}", bf_point_positive_true);
+
+    // Update _pfc_rdb_bf_point_positive_total
+    uint64_t bf_point_positive_total =
+        _statistics->getTickerCount(rocksdb::BLOOM_FILTER_FULL_POSITIVE);
+    _pfc_rdb_bf_point_positive_total->set(bf_point_positive_total);
+    dinfo_replica("_pfc_rdb_bf_point_positive_total: {}", bf_point_positive_total);
+
+    // Update _pfc_rdb_bf_point_negatives
+    uint64_t bf_point_negatives = _statistics->getTickerCount(rocksdb::BLOOM_FILTER_USEFUL);
+    _pfc_rdb_bf_point_negatives->set(bf_point_negatives);
+    dinfo_replica("_pfc_rdb_bf_point_negatives: {}", bf_point_negatives);
 }
 
 void pegasus_server_impl::update_server_rocksdb_statistics()
 {
+    // Update _pfc_rdb_block_cache_mem_usage
     if (_s_block_cache) {
         uint64_t val = _s_block_cache->GetUsage();
         _pfc_rdb_block_cache_mem_usage->set(val);

src/server/pegasus_server_impl.h

@@ -389,6 +389,11 @@ private:
     ::dsn::perf_counter_wrapper _pfc_rdb_index_and_filter_blocks_mem_usage;
     ::dsn::perf_counter_wrapper _pfc_rdb_memtable_mem_usage;
     ::dsn::perf_counter_wrapper _pfc_rdb_estimate_num_keys;
+    ::dsn::perf_counter_wrapper _pfc_rdb_bf_seek_negatives;
+    ::dsn::perf_counter_wrapper _pfc_rdb_bf_seek_total;
+    ::dsn::perf_counter_wrapper _pfc_rdb_bf_point_positive_true;
+    ::dsn::perf_counter_wrapper _pfc_rdb_bf_point_positive_total;
+    ::dsn::perf_counter_wrapper _pfc_rdb_bf_point_negatives;
 };
 
 } // namespace server

src/server/table_stats.h

@@ -56,6 +56,11 @@ struct table_stats
         total_rdb_index_and_filter_blocks_mem_usage += row.rdb_index_and_filter_blocks_mem_usage;
         total_rdb_memtable_mem_usage += row.rdb_memtable_mem_usage;
         total_rdb_estimate_num_keys += row.rdb_estimate_num_keys;
+        total_rdb_bf_seek_negatives += row.rdb_bf_seek_negatives;
+        total_rdb_bf_seek_total += row.rdb_bf_seek_total;
+        total_rdb_bf_point_positive_true += row.rdb_bf_point_positive_true;
+        total_rdb_bf_point_positive_total += row.rdb_bf_point_positive_total;
+        total_rdb_bf_point_negatives += row.rdb_bf_point_negatives;
         total_backup_request_qps += row.backup_request_qps;
         total_get_bytes += row.get_bytes;
         total_multi_get_bytes += row.multi_get_bytes;
@@ -95,6 +100,11 @@ struct table_stats
             row_stats.total_rdb_index_and_filter_blocks_mem_usage;
         total_rdb_memtable_mem_usage += row_stats.total_rdb_memtable_mem_usage;
         total_rdb_estimate_num_keys += row_stats.total_rdb_estimate_num_keys;
+        total_rdb_bf_seek_negatives += row_stats.total_rdb_bf_seek_negatives;
+        total_rdb_bf_seek_total += row_stats.total_rdb_bf_seek_total;
+        total_rdb_bf_point_positive_true += row_stats.total_rdb_bf_point_positive_true;
+        total_rdb_bf_point_positive_total += row_stats.total_rdb_bf_point_positive_total;
+        total_rdb_bf_point_negatives += row_stats.total_rdb_bf_point_negatives;
         total_backup_request_qps += row_stats.total_backup_request_qps;
         total_get_bytes += row_stats.total_get_bytes;
         total_multi_get_bytes += row_stats.total_multi_get_bytes;
@@ -130,6 +140,11 @@ struct table_stats
     double total_rdb_index_and_filter_blocks_mem_usage = 0;
     double total_rdb_memtable_mem_usage = 0;
     double total_rdb_estimate_num_keys = 0;
+    double total_rdb_bf_seek_negatives = 0;
+    double total_rdb_bf_seek_total = 0;
+    double total_rdb_bf_point_positive_true = 0;
+    double total_rdb_bf_point_positive_total = 0;
+    double total_rdb_bf_point_negatives = 0;
     double total_backup_request_qps = 0;
     double total_get_bytes = 0;
     double total_multi_get_bytes = 0;

src/shell/command_helper.h

@@ -574,6 +574,11 @@ struct row_data
     double rdb_index_and_filter_blocks_mem_usage = 0;
     double rdb_memtable_mem_usage = 0;
     double rdb_estimate_num_keys = 0;
+    double rdb_bf_seek_negatives = 0;
+    double rdb_bf_seek_total = 0;
+    double rdb_bf_point_positive_true = 0;
+    double rdb_bf_point_positive_total = 0;
+    double rdb_bf_point_negatives = 0;
     double backup_request_qps = 0;
     double get_bytes = 0;
     double multi_get_bytes = 0;
@@ -635,6 +640,16 @@ update_app_pegasus_perf_counter(row_data &row, const std::string &counter_name,
         row.rdb_memtable_mem_usage += value;
     else if (counter_name == "rdb.estimate_num_keys")
         row.rdb_estimate_num_keys += value;
+    else if (counter_name == "rdb.bf_seek_negatives")
+        row.rdb_bf_seek_negatives += value;
+    else if (counter_name == "rdb.bf_seek_total")
+        row.rdb_bf_seek_total += value;
+    else if (counter_name == "rdb.bf_point_positive_true")
+        row.rdb_bf_point_positive_true += value;
+    else if (counter_name == "rdb.bf_point_positive_total")
+        row.rdb_bf_point_positive_total += value;
+    else if (counter_name == "rdb.bf_point_negatives")
+        row.rdb_bf_point_negatives += value;
     else if (counter_name == "backup_request_qps")
         row.backup_request_qps += value;
     else if (counter_name == "get_bytes")

src/shell/commands/table_management.cpp

@@ -4,6 +4,11 @@
 
 #include "shell/commands.h"
 
+double convert_to_ratio(double hit, double total)
+{
+    return std::abs(total) < 1e-6 ? 0 : hit / total;
+}
+
 bool ls_apps(command_executor *e, shell_context *sc, arguments args)
 {
     static struct option long_options[] = {{"all", no_argument, 0, 'a'},
@@ -493,6 +498,11 @@ bool app_stat(command_executor *e, shell_context *sc, arguments args)
         sum.rdb_block_cache_total_count += row.rdb_block_cache_total_count;
         sum.rdb_index_and_filter_blocks_mem_usage += row.rdb_index_and_filter_blocks_mem_usage;
         sum.rdb_memtable_mem_usage += row.rdb_memtable_mem_usage;
+        sum.rdb_bf_seek_negatives += row.rdb_bf_seek_negatives;
+        sum.rdb_bf_seek_total += row.rdb_bf_seek_total;
+        sum.rdb_bf_point_positive_true += row.rdb_bf_point_positive_true;
+        sum.rdb_bf_point_positive_total += row.rdb_bf_point_positive_total;
+        sum.rdb_bf_point_negatives += row.rdb_bf_point_negatives;
     }
 
     std::streambuf *buf;
@@ -538,6 +548,9 @@ bool app_stat(command_executor *e, shell_context *sc, arguments args)
         tp.add_column("mem_idx_mb", tp_alignment::kRight);
     }
     tp.add_column("hit_rate", tp_alignment::kRight);
+    tp.add_column("seek_n_rate", tp_alignment::kRight);
+    tp.add_column("point_n_rate", tp_alignment::kRight);
+    tp.add_column("point_fp_rate", tp_alignment::kRight);
 
     for (row_data &row : rows) {
         tp.add_row(row.row_name);
@@ -570,11 +583,16 @@ bool app_stat(command_executor *e, shell_context *sc, arguments args)
             tp.append_data(row.rdb_memtable_mem_usage / (1 << 20U));
             tp.append_data(row.rdb_index_and_filter_blocks_mem_usage / (1 << 20U));
         }
-        double block_cache_hit_rate =
-            std::abs(row.rdb_block_cache_total_count) < 1e-6
-                ? 0.0
-                : row.rdb_block_cache_hit_count / row.rdb_block_cache_total_count;
-        tp.append_data(block_cache_hit_rate);
+        tp.append_data(
+            convert_to_ratio(row.rdb_block_cache_hit_count, row.rdb_block_cache_total_count));
+        tp.append_data(convert_to_ratio(row.rdb_bf_seek_negatives, row.rdb_bf_seek_total));
+        tp.append_data(
+            convert_to_ratio(row.rdb_bf_point_negatives,
+                             row.rdb_bf_point_negatives + row.rdb_bf_point_positive_total));
+        tp.append_data(
+            convert_to_ratio(row.rdb_bf_point_positive_total - row.rdb_bf_point_positive_true,
+                             (row.rdb_bf_point_positive_total - row.rdb_bf_point_positive_true) +
+                                 row.rdb_bf_point_negatives));
     }
     tp.output(out, json ? tp_output_format::kJsonPretty : tp_output_format::kTabular);