internal_stats.cc 53.9 KB
Newer Older
S
Siying Dong 已提交
1 2 3
//  This source code is licensed under both the GPLv2 (found in the
//  COPYING file in the root directory) and Apache 2.0 License
//  (found in the LICENSE.Apache file in the root directory).
I
Igor Canadi 已提交
4 5 6 7 8 9
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.

#include "db/internal_stats.h"
L
liuhuahang 已提交
10 11

#ifndef __STDC_FORMAT_MACROS
I
Igor Canadi 已提交
12
#define __STDC_FORMAT_MACROS
L
liuhuahang 已提交
13 14
#endif

I
Igor Canadi 已提交
15
#include <inttypes.h>
16
#include <algorithm>
Y
Yi Wu 已提交
17 18
#include <limits>
#include <string>
19
#include <utility>
I
Igor Canadi 已提交
20
#include <vector>
21
#include "db/column_family.h"
22

23
#include "db/db_impl.h"
24
#include "util/string_util.h"
I
Igor Canadi 已提交
25 26 27

namespace rocksdb {

28
#ifndef ROCKSDB_LITE
29 30 31 32 33 34

const std::map<LevelStatType, LevelStat> InternalStats::compaction_level_stats =
    {
        {LevelStatType::NUM_FILES, LevelStat{"NumFiles", "Files"}},
        {LevelStatType::COMPACTED_FILES,
         LevelStat{"CompactedFiles", "CompactedFiles"}},
35
        {LevelStatType::SIZE_BYTES, LevelStat{"SizeBytes", "Size"}},
36
        {LevelStatType::SCORE, LevelStat{"Score", "Score"}},
37 38 39 40 41 42
        {LevelStatType::READ_GB, LevelStat{"ReadGB", "Read(GB)"}},
        {LevelStatType::RN_GB, LevelStat{"RnGB", "Rn(GB)"}},
        {LevelStatType::RNP1_GB, LevelStat{"Rnp1GB", "Rnp1(GB)"}},
        {LevelStatType::WRITE_GB, LevelStat{"WriteGB", "Write(GB)"}},
        {LevelStatType::W_NEW_GB, LevelStat{"WnewGB", "Wnew(GB)"}},
        {LevelStatType::MOVED_GB, LevelStat{"MovedGB", "Moved(GB)"}},
43
        {LevelStatType::WRITE_AMP, LevelStat{"WriteAmp", "W-Amp"}},
44 45 46 47 48
        {LevelStatType::READ_MBPS, LevelStat{"ReadMBps", "Rd(MB/s)"}},
        {LevelStatType::WRITE_MBPS, LevelStat{"WriteMBps", "Wr(MB/s)"}},
        {LevelStatType::COMP_SEC, LevelStat{"CompSec", "Comp(sec)"}},
        {LevelStatType::COMP_COUNT, LevelStat{"CompCount", "Comp(cnt)"}},
        {LevelStatType::AVG_SEC, LevelStat{"AvgSec", "Avg(sec)"}},
49 50 51 52
        {LevelStatType::KEY_IN, LevelStat{"KeyIn", "KeyIn"}},
        {LevelStatType::KEY_DROP, LevelStat{"KeyDrop", "KeyDrop"}},
};

L
Lei Jin 已提交
53 54 55
namespace {
const double kMB = 1048576.0;
const double kGB = kMB * 1024;
56
const double kMicrosInSec = 1000000.0;
L
Lei Jin 已提交
57

58
void PrintLevelStatsHeader(char* buf, size_t len, const std::string& cf_name) {
59 60 61 62 63 64 65
  int written_size =
      snprintf(buf, len, "\n** Compaction Stats [%s] **\n", cf_name.c_str());
  auto hdr = [](LevelStatType t) {
    return InternalStats::compaction_level_stats.at(t).header_name.c_str();
  };
  int line_size = snprintf(
      buf + written_size, len - written_size,
66
      "Level    %s   %s     %s %s  %s %s %s %s %s %s %s %s %s %s %s %s %s\n",
67
      // Note that we skip COMPACTED_FILES and merge it with Files column
68
      hdr(LevelStatType::NUM_FILES), hdr(LevelStatType::SIZE_BYTES),
69 70 71 72 73 74 75 76 77 78 79 80
      hdr(LevelStatType::SCORE), hdr(LevelStatType::READ_GB),
      hdr(LevelStatType::RN_GB), hdr(LevelStatType::RNP1_GB),
      hdr(LevelStatType::WRITE_GB), hdr(LevelStatType::W_NEW_GB),
      hdr(LevelStatType::MOVED_GB), hdr(LevelStatType::WRITE_AMP),
      hdr(LevelStatType::READ_MBPS), hdr(LevelStatType::WRITE_MBPS),
      hdr(LevelStatType::COMP_SEC), hdr(LevelStatType::COMP_COUNT),
      hdr(LevelStatType::AVG_SEC), hdr(LevelStatType::KEY_IN),
      hdr(LevelStatType::KEY_DROP));

  written_size += line_size;
  snprintf(buf + written_size, len - written_size, "%s\n",
           std::string(line_size, '-').c_str());
L
Lei Jin 已提交
81 82
}

83 84 85 86
void PrepareLevelStats(std::map<LevelStatType, double>* level_stats,
                       int num_files, int being_compacted,
                       double total_file_size, double score, double w_amp,
                       const InternalStats::CompactionStats& stats) {
87 88 89 90
  uint64_t bytes_read =
      stats.bytes_read_non_output_levels + stats.bytes_read_output_level;
  int64_t bytes_new =
      stats.bytes_written - stats.bytes_read_output_level;
91
  double elapsed = (stats.micros + 1) / kMicrosInSec;
L
Lei Jin 已提交
92

93 94
  (*level_stats)[LevelStatType::NUM_FILES] = num_files;
  (*level_stats)[LevelStatType::COMPACTED_FILES] = being_compacted;
95
  (*level_stats)[LevelStatType::SIZE_BYTES] = total_file_size;
96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119
  (*level_stats)[LevelStatType::SCORE] = score;
  (*level_stats)[LevelStatType::READ_GB] = bytes_read / kGB;
  (*level_stats)[LevelStatType::RN_GB] =
      stats.bytes_read_non_output_levels / kGB;
  (*level_stats)[LevelStatType::RNP1_GB] = stats.bytes_read_output_level / kGB;
  (*level_stats)[LevelStatType::WRITE_GB] = stats.bytes_written / kGB;
  (*level_stats)[LevelStatType::W_NEW_GB] = bytes_new / kGB;
  (*level_stats)[LevelStatType::MOVED_GB] = stats.bytes_moved / kGB;
  (*level_stats)[LevelStatType::WRITE_AMP] = w_amp;
  (*level_stats)[LevelStatType::READ_MBPS] = bytes_read / kMB / elapsed;
  (*level_stats)[LevelStatType::WRITE_MBPS] =
      stats.bytes_written / kMB / elapsed;
  (*level_stats)[LevelStatType::COMP_SEC] = stats.micros / kMicrosInSec;
  (*level_stats)[LevelStatType::COMP_COUNT] = stats.count;
  (*level_stats)[LevelStatType::AVG_SEC] =
      stats.count == 0 ? 0 : stats.micros / kMicrosInSec / stats.count;
  (*level_stats)[LevelStatType::KEY_IN] =
      static_cast<double>(stats.num_input_records);
  (*level_stats)[LevelStatType::KEY_DROP] =
      static_cast<double>(stats.num_dropped_records);
}

void PrintLevelStats(char* buf, size_t len, const std::string& name,
                     const std::map<LevelStatType, double>& stat_value) {
L
Lei Jin 已提交
120
  snprintf(buf, len,
121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138
           "%4s "      /*  Level */
           "%6d/%-3d " /*  Files */
           "%8s "      /*  Size */
           "%5.1f "    /*  Score */
           "%8.1f "    /*  Read(GB) */
           "%7.1f "    /*  Rn(GB) */
           "%8.1f "    /*  Rnp1(GB) */
           "%9.1f "    /*  Write(GB) */
           "%8.1f "    /*  Wnew(GB) */
           "%9.1f "    /*  Moved(GB) */
           "%5.1f "    /*  W-Amp */
           "%8.1f "    /*  Rd(MB/s) */
           "%8.1f "    /*  Wr(MB/s) */
           "%9.0f "    /*  Comp(sec) */
           "%9d "      /*  Comp(cnt) */
           "%8.3f "    /*  Avg(sec) */
           "%7s "      /*  KeyIn */
           "%6s\n",    /*  KeyDrop */
139
           name.c_str(),
140 141
           static_cast<int>(stat_value.at(LevelStatType::NUM_FILES)),
           static_cast<int>(stat_value.at(LevelStatType::COMPACTED_FILES)),
142 143 144
           BytesToHumanString(
               static_cast<uint64_t>(stat_value.at(LevelStatType::SIZE_BYTES)))
               .c_str(),
145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173
           stat_value.at(LevelStatType::SCORE),
           stat_value.at(LevelStatType::READ_GB),
           stat_value.at(LevelStatType::RN_GB),
           stat_value.at(LevelStatType::RNP1_GB),
           stat_value.at(LevelStatType::WRITE_GB),
           stat_value.at(LevelStatType::W_NEW_GB),
           stat_value.at(LevelStatType::MOVED_GB),
           stat_value.at(LevelStatType::WRITE_AMP),
           stat_value.at(LevelStatType::READ_MBPS),
           stat_value.at(LevelStatType::WRITE_MBPS),
           stat_value.at(LevelStatType::COMP_SEC),
           static_cast<int>(stat_value.at(LevelStatType::COMP_COUNT)),
           stat_value.at(LevelStatType::AVG_SEC),
           NumberToHumanString(
               static_cast<std::int64_t>(stat_value.at(LevelStatType::KEY_IN)))
               .c_str(),
           NumberToHumanString(static_cast<std::int64_t>(
                                   stat_value.at(LevelStatType::KEY_DROP)))
               .c_str());
}

void PrintLevelStats(char* buf, size_t len, const std::string& name,
                     int num_files, int being_compacted, double total_file_size,
                     double score, double w_amp,
                     const InternalStats::CompactionStats& stats) {
  std::map<LevelStatType, double> level_stats;
  PrepareLevelStats(&level_stats, num_files, being_compacted, total_file_size,
                    score, w_amp, stats);
  PrintLevelStats(buf, len, name, level_stats);
L
Lei Jin 已提交
174
}
175 176 177 178 179 180 181 182 183 184 185 186 187

// Assumes that trailing numbers represent an optional argument. This requires
// property names to not end with numbers.
std::pair<Slice, Slice> GetPropertyNameAndArg(const Slice& property) {
  Slice name = property, arg = property;
  size_t sfx_len = 0;
  while (sfx_len < property.size() &&
         isdigit(property[property.size() - sfx_len - 1])) {
    ++sfx_len;
  }
  name.remove_suffix(sfx_len);
  arg.remove_prefix(property.size() - sfx_len);
  return {name, arg};
L
Lei Jin 已提交
188
}
189
}  // anonymous namespace
L
Lei Jin 已提交
190

191 192 193
static const std::string rocksdb_prefix = "rocksdb.";

static const std::string num_files_at_level_prefix = "num-files-at-level";
194 195
static const std::string compression_ratio_at_level_prefix =
    "compression-ratio-at-level";
196
static const std::string allstats = "stats";
197 198
static const std::string sstables = "sstables";
static const std::string cfstats = "cfstats";
199 200 201
static const std::string cfstats_no_file_histogram =
    "cfstats-no-file-histogram";
static const std::string cf_file_histogram = "cf-file-histogram";
202 203 204
static const std::string dbstats = "dbstats";
static const std::string levelstats = "levelstats";
static const std::string num_immutable_mem_table = "num-immutable-mem-table";
205 206
static const std::string num_immutable_mem_table_flushed =
    "num-immutable-mem-table-flushed";
207 208 209 210 211
static const std::string mem_table_flush_pending = "mem-table-flush-pending";
static const std::string compaction_pending = "compaction-pending";
static const std::string background_errors = "background-errors";
static const std::string cur_size_active_mem_table =
                          "cur-size-active-mem-table";
212 213
static const std::string cur_size_all_mem_tables = "cur-size-all-mem-tables";
static const std::string size_all_mem_tables = "size-all-mem-tables";
214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229
static const std::string num_entries_active_mem_table =
                          "num-entries-active-mem-table";
static const std::string num_entries_imm_mem_tables =
                          "num-entries-imm-mem-tables";
static const std::string num_deletes_active_mem_table =
                          "num-deletes-active-mem-table";
static const std::string num_deletes_imm_mem_tables =
                          "num-deletes-imm-mem-tables";
static const std::string estimate_num_keys = "estimate-num-keys";
static const std::string estimate_table_readers_mem =
                          "estimate-table-readers-mem";
static const std::string is_file_deletions_enabled =
                          "is-file-deletions-enabled";
static const std::string num_snapshots = "num-snapshots";
static const std::string oldest_snapshot_time = "oldest-snapshot-time";
static const std::string num_live_versions = "num-live-versions";
230 231
static const std::string current_version_number =
    "current-super-version-number";
A
Andres Notzli 已提交
232
static const std::string estimate_live_data_size = "estimate-live-data-size";
233
static const std::string min_log_number_to_keep = "min-log-number-to-keep";
234
static const std::string base_level = "base-level";
235
static const std::string total_sst_files_size = "total-sst-files-size";
236
static const std::string live_sst_files_size = "live-sst-files-size";
237 238
static const std::string estimate_pending_comp_bytes =
    "estimate-pending-compaction-bytes";
239 240 241 242
static const std::string aggregated_table_properties =
    "aggregated-table-properties";
static const std::string aggregated_table_properties_at_level =
    aggregated_table_properties + "-at-level";
243 244
static const std::string num_running_compactions = "num-running-compactions";
static const std::string num_running_flushes = "num-running-flushes";
245 246 247
static const std::string actual_delayed_write_rate =
    "actual-delayed-write-rate";
static const std::string is_write_stopped = "is-write-stopped";
Y
Yi Wu 已提交
248
static const std::string estimate_oldest_key_time = "estimate-oldest-key-time";
249 250 251

const std::string DB::Properties::kNumFilesAtLevelPrefix =
                      rocksdb_prefix + num_files_at_level_prefix;
252 253
const std::string DB::Properties::kCompressionRatioAtLevelPrefix =
                      rocksdb_prefix + compression_ratio_at_level_prefix;
254
const std::string DB::Properties::kStats = rocksdb_prefix + allstats;
255 256
const std::string DB::Properties::kSSTables = rocksdb_prefix + sstables;
const std::string DB::Properties::kCFStats = rocksdb_prefix + cfstats;
257 258 259 260
const std::string DB::Properties::kCFStatsNoFileHistogram =
    rocksdb_prefix + cfstats_no_file_histogram;
const std::string DB::Properties::kCFFileHistogram =
    rocksdb_prefix + cf_file_histogram;
261
const std::string DB::Properties::kDBStats = rocksdb_prefix + dbstats;
262
const std::string DB::Properties::kLevelStats = rocksdb_prefix + levelstats;
263 264
const std::string DB::Properties::kNumImmutableMemTable =
                      rocksdb_prefix + num_immutable_mem_table;
265
const std::string DB::Properties::kNumImmutableMemTableFlushed =
266
    rocksdb_prefix + num_immutable_mem_table_flushed;
267 268 269 270
const std::string DB::Properties::kMemTableFlushPending =
                      rocksdb_prefix + mem_table_flush_pending;
const std::string DB::Properties::kCompactionPending =
                      rocksdb_prefix + compaction_pending;
271 272 273 274
const std::string DB::Properties::kNumRunningCompactions =
    rocksdb_prefix + num_running_compactions;
const std::string DB::Properties::kNumRunningFlushes =
    rocksdb_prefix + num_running_flushes;
275 276 277 278 279
const std::string DB::Properties::kBackgroundErrors =
                      rocksdb_prefix + background_errors;
const std::string DB::Properties::kCurSizeActiveMemTable =
                      rocksdb_prefix + cur_size_active_mem_table;
const std::string DB::Properties::kCurSizeAllMemTables =
280
    rocksdb_prefix + cur_size_all_mem_tables;
281 282
const std::string DB::Properties::kSizeAllMemTables =
    rocksdb_prefix + size_all_mem_tables;
283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302
const std::string DB::Properties::kNumEntriesActiveMemTable =
                      rocksdb_prefix + num_entries_active_mem_table;
const std::string DB::Properties::kNumEntriesImmMemTables =
                      rocksdb_prefix + num_entries_imm_mem_tables;
const std::string DB::Properties::kNumDeletesActiveMemTable =
                      rocksdb_prefix + num_deletes_active_mem_table;
const std::string DB::Properties::kNumDeletesImmMemTables =
                      rocksdb_prefix + num_deletes_imm_mem_tables;
const std::string DB::Properties::kEstimateNumKeys =
                      rocksdb_prefix + estimate_num_keys;
const std::string DB::Properties::kEstimateTableReadersMem =
                      rocksdb_prefix + estimate_table_readers_mem;
const std::string DB::Properties::kIsFileDeletionsEnabled =
                      rocksdb_prefix + is_file_deletions_enabled;
const std::string DB::Properties::kNumSnapshots =
                      rocksdb_prefix + num_snapshots;
const std::string DB::Properties::kOldestSnapshotTime =
                      rocksdb_prefix + oldest_snapshot_time;
const std::string DB::Properties::kNumLiveVersions =
                      rocksdb_prefix + num_live_versions;
303
const std::string DB::Properties::kCurrentSuperVersionNumber =
304
    rocksdb_prefix + current_version_number;
A
Andres Notzli 已提交
305 306
const std::string DB::Properties::kEstimateLiveDataSize =
                      rocksdb_prefix + estimate_live_data_size;
307 308
const std::string DB::Properties::kMinLogNumberToKeep =
    rocksdb_prefix + min_log_number_to_keep;
309 310
const std::string DB::Properties::kTotalSstFilesSize =
                      rocksdb_prefix + total_sst_files_size;
311 312
const std::string DB::Properties::kLiveSstFilesSize =
    rocksdb_prefix + live_sst_files_size;
313
const std::string DB::Properties::kBaseLevel = rocksdb_prefix + base_level;
314 315
const std::string DB::Properties::kEstimatePendingCompactionBytes =
    rocksdb_prefix + estimate_pending_comp_bytes;
316 317 318 319
const std::string DB::Properties::kAggregatedTableProperties =
    rocksdb_prefix + aggregated_table_properties;
const std::string DB::Properties::kAggregatedTablePropertiesAtLevel =
    rocksdb_prefix + aggregated_table_properties_at_level;
320 321 322 323
const std::string DB::Properties::kActualDelayedWriteRate =
    rocksdb_prefix + actual_delayed_write_rate;
const std::string DB::Properties::kIsWriteStopped =
    rocksdb_prefix + is_write_stopped;
Y
Yi Wu 已提交
324 325
const std::string DB::Properties::kEstimateOldestKeyTime =
    rocksdb_prefix + estimate_oldest_key_time;
326

327 328 329 330 331 332 333 334 335 336 337 338 339 340
const std::unordered_map<std::string, DBPropertyInfo>
    InternalStats::ppt_name_to_info = {
        {DB::Properties::kNumFilesAtLevelPrefix,
         {false, &InternalStats::HandleNumFilesAtLevel, nullptr, nullptr}},
        {DB::Properties::kCompressionRatioAtLevelPrefix,
         {false, &InternalStats::HandleCompressionRatioAtLevelPrefix, nullptr,
          nullptr}},
        {DB::Properties::kLevelStats,
         {false, &InternalStats::HandleLevelStats, nullptr, nullptr}},
        {DB::Properties::kStats,
         {false, &InternalStats::HandleStats, nullptr, nullptr}},
        {DB::Properties::kCFStats,
         {false, &InternalStats::HandleCFStats, nullptr,
          &InternalStats::HandleCFMapStats}},
341 342 343 344 345
        {DB::Properties::kCFStatsNoFileHistogram,
         {false, &InternalStats::HandleCFStatsNoFileHistogram, nullptr,
          nullptr}},
        {DB::Properties::kCFFileHistogram,
         {false, &InternalStats::HandleCFFileHistogram, nullptr, nullptr}},
346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404
        {DB::Properties::kDBStats,
         {false, &InternalStats::HandleDBStats, nullptr, nullptr}},
        {DB::Properties::kSSTables,
         {false, &InternalStats::HandleSsTables, nullptr, nullptr}},
        {DB::Properties::kAggregatedTableProperties,
         {false, &InternalStats::HandleAggregatedTableProperties, nullptr,
          nullptr}},
        {DB::Properties::kAggregatedTablePropertiesAtLevel,
         {false, &InternalStats::HandleAggregatedTablePropertiesAtLevel,
          nullptr, nullptr}},
        {DB::Properties::kNumImmutableMemTable,
         {false, nullptr, &InternalStats::HandleNumImmutableMemTable, nullptr}},
        {DB::Properties::kNumImmutableMemTableFlushed,
         {false, nullptr, &InternalStats::HandleNumImmutableMemTableFlushed,
          nullptr}},
        {DB::Properties::kMemTableFlushPending,
         {false, nullptr, &InternalStats::HandleMemTableFlushPending, nullptr}},
        {DB::Properties::kCompactionPending,
         {false, nullptr, &InternalStats::HandleCompactionPending, nullptr}},
        {DB::Properties::kBackgroundErrors,
         {false, nullptr, &InternalStats::HandleBackgroundErrors, nullptr}},
        {DB::Properties::kCurSizeActiveMemTable,
         {false, nullptr, &InternalStats::HandleCurSizeActiveMemTable,
          nullptr}},
        {DB::Properties::kCurSizeAllMemTables,
         {false, nullptr, &InternalStats::HandleCurSizeAllMemTables, nullptr}},
        {DB::Properties::kSizeAllMemTables,
         {false, nullptr, &InternalStats::HandleSizeAllMemTables, nullptr}},
        {DB::Properties::kNumEntriesActiveMemTable,
         {false, nullptr, &InternalStats::HandleNumEntriesActiveMemTable,
          nullptr}},
        {DB::Properties::kNumEntriesImmMemTables,
         {false, nullptr, &InternalStats::HandleNumEntriesImmMemTables,
          nullptr}},
        {DB::Properties::kNumDeletesActiveMemTable,
         {false, nullptr, &InternalStats::HandleNumDeletesActiveMemTable,
          nullptr}},
        {DB::Properties::kNumDeletesImmMemTables,
         {false, nullptr, &InternalStats::HandleNumDeletesImmMemTables,
          nullptr}},
        {DB::Properties::kEstimateNumKeys,
         {false, nullptr, &InternalStats::HandleEstimateNumKeys, nullptr}},
        {DB::Properties::kEstimateTableReadersMem,
         {true, nullptr, &InternalStats::HandleEstimateTableReadersMem,
          nullptr}},
        {DB::Properties::kIsFileDeletionsEnabled,
         {false, nullptr, &InternalStats::HandleIsFileDeletionsEnabled,
          nullptr}},
        {DB::Properties::kNumSnapshots,
         {false, nullptr, &InternalStats::HandleNumSnapshots, nullptr}},
        {DB::Properties::kOldestSnapshotTime,
         {false, nullptr, &InternalStats::HandleOldestSnapshotTime, nullptr}},
        {DB::Properties::kNumLiveVersions,
         {false, nullptr, &InternalStats::HandleNumLiveVersions, nullptr}},
        {DB::Properties::kCurrentSuperVersionNumber,
         {false, nullptr, &InternalStats::HandleCurrentSuperVersionNumber,
          nullptr}},
        {DB::Properties::kEstimateLiveDataSize,
         {true, nullptr, &InternalStats::HandleEstimateLiveDataSize, nullptr}},
405 406
        {DB::Properties::kMinLogNumberToKeep,
         {false, nullptr, &InternalStats::HandleMinLogNumberToKeep, nullptr}},
407 408 409 410
        {DB::Properties::kBaseLevel,
         {false, nullptr, &InternalStats::HandleBaseLevel, nullptr}},
        {DB::Properties::kTotalSstFilesSize,
         {false, nullptr, &InternalStats::HandleTotalSstFilesSize, nullptr}},
411 412
        {DB::Properties::kLiveSstFilesSize,
         {false, nullptr, &InternalStats::HandleLiveSstFilesSize, nullptr}},
413 414 415 416 417 418 419 420
        {DB::Properties::kEstimatePendingCompactionBytes,
         {false, nullptr, &InternalStats::HandleEstimatePendingCompactionBytes,
          nullptr}},
        {DB::Properties::kNumRunningFlushes,
         {false, nullptr, &InternalStats::HandleNumRunningFlushes, nullptr}},
        {DB::Properties::kNumRunningCompactions,
         {false, nullptr, &InternalStats::HandleNumRunningCompactions,
          nullptr}},
421 422 423 424 425
        {DB::Properties::kActualDelayedWriteRate,
         {false, nullptr, &InternalStats::HandleActualDelayedWriteRate,
          nullptr}},
        {DB::Properties::kIsWriteStopped,
         {false, nullptr, &InternalStats::HandleIsWriteStopped, nullptr}},
Y
Yi Wu 已提交
426 427 428
        {DB::Properties::kEstimateOldestKeyTime,
         {false, nullptr, &InternalStats::HandleEstimateOldestKeyTime,
          nullptr}},
429 430 431 432 433 434 435
};

const DBPropertyInfo* GetPropertyInfo(const Slice& property) {
  std::string ppt_name = GetPropertyNameAndArg(property).first.ToString();
  auto ppt_info_iter = InternalStats::ppt_name_to_info.find(ppt_name);
  if (ppt_info_iter == InternalStats::ppt_name_to_info.end()) {
    return nullptr;
436
  }
437 438
  return &ppt_info_iter->second;
}
439

440 441 442 443 444 445 446
bool InternalStats::GetStringProperty(const DBPropertyInfo& property_info,
                                      const Slice& property,
                                      std::string* value) {
  assert(value != nullptr);
  assert(property_info.handle_string != nullptr);
  Slice arg = GetPropertyNameAndArg(property).second;
  return (this->*(property_info.handle_string))(value, arg);
447 448
}

449
bool InternalStats::GetMapProperty(const DBPropertyInfo& property_info,
A
Andrew Kryczka 已提交
450
                                   const Slice& /*property*/,
451
                                   std::map<std::string, std::string>* value) {
452 453 454 455 456
  assert(value != nullptr);
  assert(property_info.handle_map != nullptr);
  return (this->*(property_info.handle_map))(value);
}

457 458
bool InternalStats::GetIntProperty(const DBPropertyInfo& property_info,
                                   uint64_t* value, DBImpl* db) {
459
  assert(value != nullptr);
460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475
  assert(property_info.handle_int != nullptr &&
         !property_info.need_out_of_mutex);
  db->mutex_.AssertHeld();
  return (this->*(property_info.handle_int))(value, db, nullptr /* version */);
}

bool InternalStats::GetIntPropertyOutOfMutex(
    const DBPropertyInfo& property_info, Version* version, uint64_t* value) {
  assert(value != nullptr);
  assert(property_info.handle_int != nullptr &&
         property_info.need_out_of_mutex);
  return (this->*(property_info.handle_int))(value, nullptr /* db */, version);
}

bool InternalStats::HandleNumFilesAtLevel(std::string* value, Slice suffix) {
  uint64_t level;
A
Andres Notzli 已提交
476
  const auto* vstorage = cfd_->current()->storage_info();
477 478 479 480 481 482 483 484 485 486 487 488
  bool ok = ConsumeDecimalNumber(&suffix, &level) && suffix.empty();
  if (!ok || static_cast<int>(level) >= number_levels_) {
    return false;
  } else {
    char buf[100];
    snprintf(buf, sizeof(buf), "%d",
             vstorage->NumLevelFiles(static_cast<int>(level)));
    *value = buf;
    return true;
  }
}

489 490 491 492 493 494 495 496 497 498 499 500 501
bool InternalStats::HandleCompressionRatioAtLevelPrefix(std::string* value,
                                                        Slice suffix) {
  uint64_t level;
  const auto* vstorage = cfd_->current()->storage_info();
  bool ok = ConsumeDecimalNumber(&suffix, &level) && suffix.empty();
  if (!ok || level >= static_cast<uint64_t>(number_levels_)) {
    return false;
  }
  *value = ToString(
      vstorage->GetEstimatedCompressionRatioAtLevel(static_cast<int>(level)));
  return true;
}

A
Andrew Kryczka 已提交
502
bool InternalStats::HandleLevelStats(std::string* value, Slice /*suffix*/) {
503 504 505 506 507 508
  char buf[1000];
  const auto* vstorage = cfd_->current()->storage_info();
  snprintf(buf, sizeof(buf),
           "Level Files Size(MB)\n"
           "--------------------\n");
  value->append(buf);
A
Andres Notzli 已提交
509

510 511 512 513 514
  for (int level = 0; level < number_levels_; level++) {
    snprintf(buf, sizeof(buf), "%3d %8d %8.0f\n", level,
             vstorage->NumLevelFiles(level),
             vstorage->NumLevelBytes(level) / kMB);
    value->append(buf);
515
  }
516
  return true;
517
}
518

519 520 521 522 523 524 525 526 527 528
bool InternalStats::HandleStats(std::string* value, Slice suffix) {
  if (!HandleCFStats(value, suffix)) {
    return false;
  }
  if (!HandleDBStats(value, suffix)) {
    return false;
  }
  return true;
}

529 530
bool InternalStats::HandleCFMapStats(
    std::map<std::string, std::string>* cf_stats) {
531 532 533 534
  DumpCFMapStats(cf_stats);
  return true;
}

A
Andrew Kryczka 已提交
535
bool InternalStats::HandleCFStats(std::string* value, Slice /*suffix*/) {
536 537 538 539
  DumpCFStats(value);
  return true;
}

540
bool InternalStats::HandleCFStatsNoFileHistogram(std::string* value,
A
Andrew Kryczka 已提交
541
                                                 Slice /*suffix*/) {
542 543 544 545
  DumpCFStatsNoFileHistogram(value);
  return true;
}

A
Andrew Kryczka 已提交
546 547
bool InternalStats::HandleCFFileHistogram(std::string* value,
                                          Slice /*suffix*/) {
548 549 550 551
  DumpCFFileHistogram(value);
  return true;
}

A
Andrew Kryczka 已提交
552
bool InternalStats::HandleDBStats(std::string* value, Slice /*suffix*/) {
553 554 555 556
  DumpDBStats(value);
  return true;
}

A
Andrew Kryczka 已提交
557
bool InternalStats::HandleSsTables(std::string* value, Slice /*suffix*/) {
S
sdong 已提交
558
  auto* current = cfd_->current();
559
  *value = current->DebugString(true, true);
560 561
  return true;
}
I
Igor Canadi 已提交
562

563
bool InternalStats::HandleAggregatedTableProperties(std::string* value,
A
Andrew Kryczka 已提交
564
                                                    Slice /*suffix*/) {
565 566 567 568
  std::shared_ptr<const TableProperties> tp;
  auto s = cfd_->current()->GetAggregatedTableProperties(&tp);
  if (!s.ok()) {
    return false;
569
  }
570 571
  *value = tp->ToString();
  return true;
572 573
}

574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590
bool InternalStats::HandleAggregatedTablePropertiesAtLevel(std::string* value,
                                                           Slice suffix) {
  uint64_t level;
  bool ok = ConsumeDecimalNumber(&suffix, &level) && suffix.empty();
  if (!ok || static_cast<int>(level) >= number_levels_) {
    return false;
  }
  std::shared_ptr<const TableProperties> tp;
  auto s = cfd_->current()->GetAggregatedTableProperties(
      &tp, static_cast<int>(level));
  if (!s.ok()) {
    return false;
  }
  *value = tp->ToString();
  return true;
}

A
Andrew Kryczka 已提交
591 592
bool InternalStats::HandleNumImmutableMemTable(uint64_t* value, DBImpl* /*db*/,
                                               Version* /*version*/) {
593 594 595 596 597
  *value = cfd_->imm()->NumNotFlushed();
  return true;
}

bool InternalStats::HandleNumImmutableMemTableFlushed(uint64_t* value,
A
Andrew Kryczka 已提交
598 599
                                                      DBImpl* /*db*/,
                                                      Version* /*version*/) {
600 601 602 603
  *value = cfd_->imm()->NumFlushed();
  return true;
}

A
Andrew Kryczka 已提交
604 605
bool InternalStats::HandleMemTableFlushPending(uint64_t* value, DBImpl* /*db*/,
                                               Version* /*version*/) {
606 607 608 609 610 611
  // Return number of mem tables that are ready to flush (made immutable)
  *value = (cfd_->imm()->IsFlushPending() ? 1 : 0);
  return true;
}

bool InternalStats::HandleNumRunningFlushes(uint64_t* value, DBImpl* db,
A
Andrew Kryczka 已提交
612
                                            Version* /*version*/) {
613 614 615 616
  *value = db->num_running_flushes();
  return true;
}

A
Andrew Kryczka 已提交
617 618
bool InternalStats::HandleCompactionPending(uint64_t* value, DBImpl* /*db*/,
                                            Version* /*version*/) {
619 620
  // 1 if the system already determines at least one compaction is needed.
  // 0 otherwise,
S
sdong 已提交
621
  const auto* vstorage = cfd_->current()->storage_info();
622 623 624
  *value = (cfd_->compaction_picker()->NeedsCompaction(vstorage) ? 1 : 0);
  return true;
}
625

626
bool InternalStats::HandleNumRunningCompactions(uint64_t* value, DBImpl* db,
A
Andrew Kryczka 已提交
627
                                                Version* /*version*/) {
628 629 630 631
  *value = db->num_running_compactions_;
  return true;
}

A
Andrew Kryczka 已提交
632 633
bool InternalStats::HandleBackgroundErrors(uint64_t* value, DBImpl* /*db*/,
                                           Version* /*version*/) {
634 635 636 637 638
  // Accumulated number of  errors in background flushes or compactions.
  *value = GetBackgroundErrorCount();
  return true;
}

A
Andrew Kryczka 已提交
639 640
bool InternalStats::HandleCurSizeActiveMemTable(uint64_t* value, DBImpl* /*db*/,
                                                Version* /*version*/) {
641 642 643 644 645
  // Current size of the active memtable
  *value = cfd_->mem()->ApproximateMemoryUsage();
  return true;
}

A
Andrew Kryczka 已提交
646 647
bool InternalStats::HandleCurSizeAllMemTables(uint64_t* value, DBImpl* /*db*/,
                                              Version* /*version*/) {
648 649 650 651 652 653
  // Current size of the active memtable + immutable memtables
  *value = cfd_->mem()->ApproximateMemoryUsage() +
           cfd_->imm()->ApproximateUnflushedMemTablesMemoryUsage();
  return true;
}

A
Andrew Kryczka 已提交
654 655
bool InternalStats::HandleSizeAllMemTables(uint64_t* value, DBImpl* /*db*/,
                                           Version* /*version*/) {
656 657 658 659 660
  *value = cfd_->mem()->ApproximateMemoryUsage() +
           cfd_->imm()->ApproximateMemoryUsage();
  return true;
}

A
Andrew Kryczka 已提交
661 662 663
bool InternalStats::HandleNumEntriesActiveMemTable(uint64_t* value,
                                                   DBImpl* /*db*/,
                                                   Version* /*version*/) {
664 665 666 667 668
  // Current number of entires in the active memtable
  *value = cfd_->mem()->num_entries();
  return true;
}

A
Andrew Kryczka 已提交
669 670 671
bool InternalStats::HandleNumEntriesImmMemTables(uint64_t* value,
                                                 DBImpl* /*db*/,
                                                 Version* /*version*/) {
672 673 674 675 676
  // Current number of entries in the immutable memtables
  *value = cfd_->imm()->current()->GetTotalNumEntries();
  return true;
}

A
Andrew Kryczka 已提交
677 678 679
bool InternalStats::HandleNumDeletesActiveMemTable(uint64_t* value,
                                                   DBImpl* /*db*/,
                                                   Version* /*version*/) {
680 681 682 683 684
  // Current number of entires in the active memtable
  *value = cfd_->mem()->num_deletes();
  return true;
}

A
Andrew Kryczka 已提交
685 686 687
bool InternalStats::HandleNumDeletesImmMemTables(uint64_t* value,
                                                 DBImpl* /*db*/,
                                                 Version* /*version*/) {
688 689 690 691 692
  // Current number of entries in the immutable memtables
  *value = cfd_->imm()->current()->GetTotalNumDeletes();
  return true;
}

A
Andrew Kryczka 已提交
693 694
bool InternalStats::HandleEstimateNumKeys(uint64_t* value, DBImpl* /*db*/,
                                          Version* /*version*/) {
695 696 697
  // Estimate number of entries in the column family:
  // Use estimated entries in tables + total entries in memtables.
  const auto* vstorage = cfd_->current()->storage_info();
698 699 700 701 702 703 704 705
  uint64_t estimate_keys = cfd_->mem()->num_entries() +
                           cfd_->imm()->current()->GetTotalNumEntries() +
                           vstorage->GetEstimatedActiveKeys();
  uint64_t estimate_deletes =
      cfd_->mem()->num_deletes() + cfd_->imm()->current()->GetTotalNumDeletes();
  *value = estimate_keys > estimate_deletes * 2
               ? estimate_keys - (estimate_deletes * 2)
               : 0;
706 707 708 709
  return true;
}

bool InternalStats::HandleNumSnapshots(uint64_t* value, DBImpl* db,
A
Andrew Kryczka 已提交
710
                                       Version* /*version*/) {
711 712 713 714 715
  *value = db->snapshots().count();
  return true;
}

bool InternalStats::HandleOldestSnapshotTime(uint64_t* value, DBImpl* db,
A
Andrew Kryczka 已提交
716
                                             Version* /*version*/) {
717 718 719 720
  *value = static_cast<uint64_t>(db->snapshots().GetOldestSnapshotTime());
  return true;
}

A
Andrew Kryczka 已提交
721 722
bool InternalStats::HandleNumLiveVersions(uint64_t* value, DBImpl* /*db*/,
                                          Version* /*version*/) {
723 724 725 726
  *value = cfd_->GetNumLiveVersions();
  return true;
}

A
Andrew Kryczka 已提交
727 728 729
bool InternalStats::HandleCurrentSuperVersionNumber(uint64_t* value,
                                                    DBImpl* /*db*/,
                                                    Version* /*version*/) {
730 731 732 733
  *value = cfd_->GetSuperVersionNumber();
  return true;
}

734
bool InternalStats::HandleIsFileDeletionsEnabled(uint64_t* value, DBImpl* db,
A
Andrew Kryczka 已提交
735
                                                 Version* /*version*/) {
736 737 738 739
  *value = db->IsFileDeletionsEnabled();
  return true;
}

A
Andrew Kryczka 已提交
740 741
bool InternalStats::HandleBaseLevel(uint64_t* value, DBImpl* /*db*/,
                                    Version* /*version*/) {
742 743 744 745 746
  const auto* vstorage = cfd_->current()->storage_info();
  *value = vstorage->base_level();
  return true;
}

747 748
bool InternalStats::HandleTotalSstFilesSize(uint64_t* value, DBImpl* /*db*/,
                                            Version* /*version*/) {
749 750 751 752
  *value = cfd_->GetTotalSstFilesSize();
  return true;
}

753 754 755 756 757 758
bool InternalStats::HandleLiveSstFilesSize(uint64_t* value, DBImpl* /*db*/,
                                           Version* /*version*/) {
  *value = cfd_->GetLiveSstFilesSize();
  return true;
}

759
bool InternalStats::HandleEstimatePendingCompactionBytes(uint64_t* value,
A
Andrew Kryczka 已提交
760 761
                                                         DBImpl* /*db*/,
                                                         Version* /*version*/) {
762 763 764 765 766
  const auto* vstorage = cfd_->current()->storage_info();
  *value = vstorage->estimated_compaction_needed_bytes();
  return true;
}

A
Andrew Kryczka 已提交
767 768
bool InternalStats::HandleEstimateTableReadersMem(uint64_t* value,
                                                  DBImpl* /*db*/,
769 770 771 772 773
                                                  Version* version) {
  *value = (version == nullptr) ? 0 : version->GetMemoryUsageByTableReaders();
  return true;
}

A
Andrew Kryczka 已提交
774 775
bool InternalStats::HandleEstimateLiveDataSize(uint64_t* value, DBImpl* /*db*/,
                                               Version* /*version*/) {
776 777 778
  const auto* vstorage = cfd_->current()->storage_info();
  *value = vstorage->EstimateLiveDataSize();
  return true;
I
Igor Canadi 已提交
779 780
}

781
bool InternalStats::HandleMinLogNumberToKeep(uint64_t* value, DBImpl* db,
A
Andrew Kryczka 已提交
782
                                             Version* /*version*/) {
783 784 785 786
  *value = db->MinLogNumberToKeep();
  return true;
}

787
bool InternalStats::HandleActualDelayedWriteRate(uint64_t* value, DBImpl* db,
A
Andrew Kryczka 已提交
788
                                                 Version* /*version*/) {
789 790 791 792 793 794 795 796 797 798
  const WriteController& wc = db->write_controller();
  if (!wc.NeedsDelay()) {
    *value = 0;
  } else {
    *value = wc.delayed_write_rate();
  }
  return true;
}

bool InternalStats::HandleIsWriteStopped(uint64_t* value, DBImpl* db,
A
Andrew Kryczka 已提交
799
                                         Version* /*version*/) {
800 801 802 803
  *value = db->write_controller().IsStopped() ? 1 : 0;
  return true;
}

Y
Yi Wu 已提交
804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822
bool InternalStats::HandleEstimateOldestKeyTime(uint64_t* value, DBImpl* /*db*/,
                                                Version* /*version*/) {
  // TODO(yiwu): The property is currently available for fifo compaction
  // with allow_compaction = false. This is because we don't propagate
  // oldest_key_time on compaction.
  if (cfd_->ioptions()->compaction_style != kCompactionStyleFIFO ||
      cfd_->GetCurrentMutableCFOptions()
          ->compaction_options_fifo.allow_compaction) {
    return false;
  }

  TablePropertiesCollection collection;
  auto s = cfd_->current()->GetPropertiesOfAllTables(&collection);
  if (!s.ok()) {
    return false;
  }
  *value = std::numeric_limits<uint64_t>::max();
  for (auto& p : collection) {
    *value = std::min(*value, p.second->oldest_key_time);
823 824 825
    if (*value == 0) {
      break;
    }
Y
Yi Wu 已提交
826
  }
827 828 829 830 831
  if (*value > 0) {
    *value = std::min({cfd_->mem()->ApproximateOldestKeyTime(),
                       cfd_->imm()->ApproximateOldestKeyTime(), *value});
  }
  return *value > 0 && *value < std::numeric_limits<uint64_t>::max();
Y
Yi Wu 已提交
832 833
}

834 835 836
void InternalStats::DumpDBStats(std::string* value) {
  char buf[1000];
  // DB-level stats, only available from default column family
837
  double seconds_up = (env_->NowMicros() - started_at_ + 1) / kMicrosInSec;
838 839 840 841 842 843
  double interval_seconds_up = seconds_up - db_stats_snapshot_.seconds_up;
  snprintf(buf, sizeof(buf),
           "\n** DB Stats **\nUptime(secs): %.1f total, %.1f interval\n",
           seconds_up, interval_seconds_up);
  value->append(buf);
  // Cumulative
844 845 846 847 848 849 850 851
  uint64_t user_bytes_written = GetDBStats(InternalStats::BYTES_WRITTEN);
  uint64_t num_keys_written = GetDBStats(InternalStats::NUMBER_KEYS_WRITTEN);
  uint64_t write_other = GetDBStats(InternalStats::WRITE_DONE_BY_OTHER);
  uint64_t write_self = GetDBStats(InternalStats::WRITE_DONE_BY_SELF);
  uint64_t wal_bytes = GetDBStats(InternalStats::WAL_FILE_BYTES);
  uint64_t wal_synced = GetDBStats(InternalStats::WAL_FILE_SYNCED);
  uint64_t write_with_wal = GetDBStats(InternalStats::WRITE_WITH_WAL);
  uint64_t write_stall_micros = GetDBStats(InternalStats::WRITE_STALL_MICROS);
852

853 854 855
  const int kHumanMicrosLen = 32;
  char human_micros[kHumanMicrosLen];

856
  // Data
S
sdong 已提交
857 858
  // writes: total number of write requests.
  // keys: total number of key updates issued by all the write requests
859 860
  // commit groups: number of group commits issued to the DB. Each group can
  //                contain one or more writes.
S
sdong 已提交
861
  // so writes/keys is the average number of put in multi-put or put
862
  // writes/groups is the average group commit size.
S
sdong 已提交
863 864
  //
  // The format is the same for interval stats.
865
  snprintf(buf, sizeof(buf),
866 867
           "Cumulative writes: %s writes, %s keys, %s commit groups, "
           "%.1f writes per commit group, ingest: %.2f GB, %.2f MB/s\n",
868 869 870
           NumberToHumanString(write_other + write_self).c_str(),
           NumberToHumanString(num_keys_written).c_str(),
           NumberToHumanString(write_self).c_str(),
871
           (write_other + write_self) / static_cast<double>(write_self + 1),
872
           user_bytes_written / kGB, user_bytes_written / kMB / seconds_up);
873 874 875
  value->append(buf);
  // WAL
  snprintf(buf, sizeof(buf),
876 877 878 879
           "Cumulative WAL: %s writes, %s syncs, "
           "%.2f writes per sync, written: %.2f GB, %.2f MB/s\n",
           NumberToHumanString(write_with_wal).c_str(),
           NumberToHumanString(wal_synced).c_str(),
880
           write_with_wal / static_cast<double>(wal_synced + 1),
881 882 883 884 885 886 887 888 889
           wal_bytes / kGB, wal_bytes / kMB / seconds_up);
  value->append(buf);
  // Stall
  AppendHumanMicros(write_stall_micros, human_micros, kHumanMicrosLen, true);
  snprintf(buf, sizeof(buf),
           "Cumulative stall: %s, %.1f percent\n",
           human_micros,
           // 10000 = divide by 1M to get secs, then multiply by 100 for pct
           write_stall_micros / 10000.0 / std::max(seconds_up, 0.001));
890 891 892 893 894
  value->append(buf);

  // Interval
  uint64_t interval_write_other = write_other - db_stats_snapshot_.write_other;
  uint64_t interval_write_self = write_self - db_stats_snapshot_.write_self;
S
sdong 已提交
895 896
  uint64_t interval_num_keys_written =
      num_keys_written - db_stats_snapshot_.num_keys_written;
897
  snprintf(buf, sizeof(buf),
898 899
           "Interval writes: %s writes, %s keys, %s commit groups, "
           "%.1f writes per commit group, ingest: %.2f MB, %.2f MB/s\n",
900 901 902 903
           NumberToHumanString(
               interval_write_other + interval_write_self).c_str(),
           NumberToHumanString(interval_num_keys_written).c_str(),
           NumberToHumanString(interval_write_self).c_str(),
904 905
           static_cast<double>(interval_write_other + interval_write_self) /
               (interval_write_self + 1),
S
sdong 已提交
906
           (user_bytes_written - db_stats_snapshot_.ingest_bytes) / kMB,
907 908
           (user_bytes_written - db_stats_snapshot_.ingest_bytes) / kMB /
               std::max(interval_seconds_up, 0.001)),
909 910 911 912 913 914 915 916
  value->append(buf);

  uint64_t interval_write_with_wal =
      write_with_wal - db_stats_snapshot_.write_with_wal;
  uint64_t interval_wal_synced = wal_synced - db_stats_snapshot_.wal_synced;
  uint64_t interval_wal_bytes = wal_bytes - db_stats_snapshot_.wal_bytes;

  snprintf(buf, sizeof(buf),
917 918 919 920
           "Interval WAL: %s writes, %s syncs, "
           "%.2f writes per sync, written: %.2f MB, %.2f MB/s\n",
           NumberToHumanString(interval_write_with_wal).c_str(),
           NumberToHumanString(interval_wal_synced).c_str(),
921 922
           interval_write_with_wal /
              static_cast<double>(interval_wal_synced + 1),
923 924 925 926 927 928 929 930 931 932 933 934 935 936
           interval_wal_bytes / kGB,
           interval_wal_bytes / kMB / std::max(interval_seconds_up, 0.001));
  value->append(buf);

  // Stall
  AppendHumanMicros(
      write_stall_micros - db_stats_snapshot_.write_stall_micros,
      human_micros, kHumanMicrosLen, true);
  snprintf(buf, sizeof(buf),
           "Interval stall: %s, %.1f percent\n",
           human_micros,
           // 10000 = divide by 1M to get secs, then multiply by 100 for pct
           (write_stall_micros - db_stats_snapshot_.write_stall_micros) /
               10000.0 / std::max(interval_seconds_up, 0.001));
937 938 939 940 941 942
  value->append(buf);

  db_stats_snapshot_.seconds_up = seconds_up;
  db_stats_snapshot_.ingest_bytes = user_bytes_written;
  db_stats_snapshot_.write_other = write_other;
  db_stats_snapshot_.write_self = write_self;
S
sdong 已提交
943
  db_stats_snapshot_.num_keys_written = num_keys_written;
944 945 946
  db_stats_snapshot_.wal_bytes = wal_bytes;
  db_stats_snapshot_.wal_synced = wal_synced;
  db_stats_snapshot_.write_with_wal = write_with_wal;
S
sdong 已提交
947
  db_stats_snapshot_.write_stall_micros = write_stall_micros;
948 949
}

950
/**
951 952 953 954 955 956
 * Dump Compaction Level stats to a map of stat name with "compaction." prefix
 * to value in double as string. The level in stat name is represented with
 * a prefix "Lx" where "x" is the level number. A special level "Sum"
 * represents the sum of a stat for all levels.
 * The result also contains IO stall counters which keys start with "io_stalls."
 * and values represent uint64 encoded as strings.
957
 */
958 959
void InternalStats::DumpCFMapStats(
        std::map<std::string, std::string>* cf_stats) {
960 961 962 963 964 965 966 967 968
  CompactionStats compaction_stats_sum(0);
  std::map<int, std::map<LevelStatType, double>> levels_stats;
  DumpCFMapStats(&levels_stats, &compaction_stats_sum);
  for (auto const& level_ent : levels_stats) {
    auto level_str =
        level_ent.first == -1 ? "Sum" : "L" + ToString(level_ent.first);
    for (auto const& stat_ent : level_ent.second) {
      auto stat_type = stat_ent.first;
      auto key_str =
969
          "compaction." + level_str + "." +
970
          InternalStats::compaction_level_stats.at(stat_type).property_name;
971
      (*cf_stats)[key_str] = std::to_string(stat_ent.second);
972 973
    }
  }
974 975

  DumpCFMapStatsIOStalls(cf_stats);
976 977
}

978
void InternalStats::DumpCFMapStats(
979 980
    std::map<int, std::map<LevelStatType, double>>* levels_stats,
    CompactionStats* compaction_stats_sum) {
S
sdong 已提交
981
  const VersionStorageInfo* vstorage = cfd_->current()->storage_info();
982 983

  int num_levels_to_check =
984
      (cfd_->ioptions()->compaction_style != kCompactionStyleFIFO)
985
          ? vstorage->num_levels() - 1
986
          : 1;
S
sdong 已提交
987

988
  // Compaction scores are sorted based on its value. Restore them to the
989 990 991
  // level order
  std::vector<double> compaction_score(number_levels_, 0);
  for (int i = 0; i < num_levels_to_check; ++i) {
992 993
    compaction_score[vstorage->CompactionScoreLevel(i)] =
        vstorage->CompactionScore(i);
994 995 996
  }
  // Count # of files being compacted for each level
  std::vector<int> files_being_compacted(number_levels_, 0);
997
  for (int level = 0; level < number_levels_; ++level) {
998
    for (auto* f : vstorage->LevelFiles(level)) {
999 1000 1001 1002 1003 1004 1005 1006 1007
      if (f->being_compacted) {
        ++files_being_compacted[level];
      }
    }
  }

  int total_files = 0;
  int total_files_being_compacted = 0;
  double total_file_size = 0;
1008 1009 1010
  uint64_t flush_ingest = cf_stats_value_[BYTES_FLUSHED];
  uint64_t add_file_ingest = cf_stats_value_[BYTES_INGESTED_ADD_FILE];
  uint64_t curr_ingest = flush_ingest + add_file_ingest;
1011
  for (int level = 0; level < number_levels_; level++) {
S
sdong 已提交
1012
    int files = vstorage->NumLevelFiles(level);
1013 1014 1015
    total_files += files;
    total_files_being_compacted += files_being_compacted[level];
    if (comp_stats_[level].micros > 0 || files > 0) {
1016
      compaction_stats_sum->Add(comp_stats_[level]);
S
sdong 已提交
1017
      total_file_size += vstorage->NumLevelBytes(level);
1018 1019 1020 1021 1022 1023
      uint64_t input_bytes;
      if (level == 0) {
        input_bytes = curr_ingest;
      } else {
        input_bytes = comp_stats_[level].bytes_read_non_output_levels;
      }
1024
      double w_amp =
1025
          (input_bytes == 0)
1026 1027
              ? 0.0
              : static_cast<double>(comp_stats_[level].bytes_written) /
1028
                    input_bytes;
1029 1030 1031 1032 1033
      std::map<LevelStatType, double> level_stats;
      PrepareLevelStats(&level_stats, files, files_being_compacted[level],
                        static_cast<double>(vstorage->NumLevelBytes(level)),
                        compaction_score[level], w_amp, comp_stats_[level]);
      (*levels_stats)[level] = level_stats;
1034 1035
    }
  }
1036 1037 1038 1039 1040 1041 1042 1043 1044 1045
  // Cumulative summary
  double w_amp = compaction_stats_sum->bytes_written /
                 static_cast<double>(curr_ingest + 1);
  // Stats summary across levels
  std::map<LevelStatType, double> sum_stats;
  PrepareLevelStats(&sum_stats, total_files, total_files_being_compacted,
                    total_file_size, 0, w_amp, *compaction_stats_sum);
  (*levels_stats)[-1] = sum_stats;  //  -1 is for the Sum level
}

1046 1047 1048
void InternalStats::DumpCFMapStatsIOStalls(
    std::map<std::string, std::string>* cf_stats) {
  (*cf_stats)["io_stalls.level0_slowdown"] =
1049
      std::to_string(cf_stats_count_[L0_FILE_COUNT_LIMIT_SLOWDOWNS]);
1050
  (*cf_stats)["io_stalls.level0_slowdown_with_compaction"] =
1051
      std::to_string(cf_stats_count_[LOCKED_L0_FILE_COUNT_LIMIT_SLOWDOWNS]);
1052
  (*cf_stats)["io_stalls.level0_numfiles"] =
1053
      std::to_string(cf_stats_count_[L0_FILE_COUNT_LIMIT_STOPS]);
1054
  (*cf_stats)["io_stalls.level0_numfiles_with_compaction"] =
1055
      std::to_string(cf_stats_count_[LOCKED_L0_FILE_COUNT_LIMIT_STOPS]);
1056
  (*cf_stats)["io_stalls.stop_for_pending_compaction_bytes"] =
1057
      std::to_string(cf_stats_count_[PENDING_COMPACTION_BYTES_LIMIT_STOPS]);
1058
  (*cf_stats)["io_stalls.slowdown_for_pending_compaction_bytes"] =
1059
      std::to_string(cf_stats_count_[PENDING_COMPACTION_BYTES_LIMIT_SLOWDOWNS]);
1060
  (*cf_stats)["io_stalls.memtable_compaction"] =
1061
      std::to_string(cf_stats_count_[MEMTABLE_LIMIT_STOPS]);
1062
  (*cf_stats)["io_stalls.memtable_slowdown"] =
1063
      std::to_string(cf_stats_count_[MEMTABLE_LIMIT_SLOWDOWNS]);
1064

1065 1066 1067
  uint64_t total_stop = cf_stats_count_[L0_FILE_COUNT_LIMIT_STOPS] +
                        cf_stats_count_[PENDING_COMPACTION_BYTES_LIMIT_STOPS] +
                        cf_stats_count_[MEMTABLE_LIMIT_STOPS];
1068 1069

  uint64_t total_slowdown =
1070 1071 1072
      cf_stats_count_[L0_FILE_COUNT_LIMIT_SLOWDOWNS] +
      cf_stats_count_[PENDING_COMPACTION_BYTES_LIMIT_SLOWDOWNS] +
      cf_stats_count_[MEMTABLE_LIMIT_SLOWDOWNS];
1073 1074 1075 1076 1077

  (*cf_stats)["io_stalls.total_stop"] = std::to_string(total_stop);
  (*cf_stats)["io_stalls.total_slowdown"] = std::to_string(total_slowdown);
}

1078
void InternalStats::DumpCFStats(std::string* value) {
1079 1080 1081 1082 1083
  DumpCFStatsNoFileHistogram(value);
  DumpCFFileHistogram(value);
}

void InternalStats::DumpCFStatsNoFileHistogram(std::string* value) {
J
Jonathan Lee 已提交
1084
  char buf[2000];
1085 1086 1087 1088 1089 1090 1091
  // Per-ColumnFamily stats
  PrintLevelStatsHeader(buf, sizeof(buf), cfd_->GetName());
  value->append(buf);

  // Print stats for each level
  std::map<int, std::map<LevelStatType, double>> levels_stats;
  CompactionStats compaction_stats_sum(0);
1092 1093
  DumpCFMapStats(&levels_stats, &compaction_stats_sum);
  for (int l = 0; l < number_levels_; ++l) {
1094 1095 1096 1097 1098
    if (levels_stats.find(l) != levels_stats.end()) {
      PrintLevelStats(buf, sizeof(buf), "L" + ToString(l), levels_stats[l]);
      value->append(buf);
    }
  }
1099

1100 1101 1102 1103 1104 1105
  // Print sum of level stats
  PrintLevelStats(buf, sizeof(buf), "Sum", levels_stats[-1]);
  value->append(buf);

  uint64_t flush_ingest = cf_stats_value_[BYTES_FLUSHED];
  uint64_t add_file_ingest = cf_stats_value_[BYTES_INGESTED_ADD_FILE];
1106 1107 1108 1109
  uint64_t ingest_files_addfile = cf_stats_value_[INGESTED_NUM_FILES_TOTAL];
  uint64_t ingest_l0_files_addfile =
      cf_stats_value_[INGESTED_LEVEL0_NUM_FILES_TOTAL];
  uint64_t ingest_keys_addfile = cf_stats_value_[INGESTED_NUM_KEYS_TOTAL];
1110
  // Cumulative summary
1111
  uint64_t total_stall_count =
1112 1113 1114 1115 1116 1117
      cf_stats_count_[L0_FILE_COUNT_LIMIT_SLOWDOWNS] +
      cf_stats_count_[L0_FILE_COUNT_LIMIT_STOPS] +
      cf_stats_count_[PENDING_COMPACTION_BYTES_LIMIT_SLOWDOWNS] +
      cf_stats_count_[PENDING_COMPACTION_BYTES_LIMIT_STOPS] +
      cf_stats_count_[MEMTABLE_LIMIT_STOPS] +
      cf_stats_count_[MEMTABLE_LIMIT_SLOWDOWNS];
1118
  // Interval summary
1119 1120 1121
  uint64_t interval_flush_ingest =
      flush_ingest - cf_stats_snapshot_.ingest_bytes_flush;
  uint64_t interval_add_file_inget =
1122
      add_file_ingest - cf_stats_snapshot_.ingest_bytes_addfile;
1123
  uint64_t interval_ingest =
1124
      interval_flush_ingest + interval_add_file_inget + 1;
1125
  CompactionStats interval_stats(compaction_stats_sum);
1126
  interval_stats.Subtract(cf_stats_snapshot_.comp_stats);
1127 1128
  double w_amp =
      interval_stats.bytes_written / static_cast<double>(interval_ingest);
1129
  PrintLevelStats(buf, sizeof(buf), "Int", 0, 0, 0, 0, w_amp, interval_stats);
1130 1131
  value->append(buf);

1132 1133 1134 1135 1136
  double seconds_up = (env_->NowMicros() - started_at_ + 1) / kMicrosInSec;
  double interval_seconds_up = seconds_up - cf_stats_snapshot_.seconds_up;
  snprintf(buf, sizeof(buf), "Uptime(secs): %.1f total, %.1f interval\n",
           seconds_up, interval_seconds_up);
  value->append(buf);
1137 1138
  snprintf(buf, sizeof(buf), "Flush(GB): cumulative %.3f, interval %.3f\n",
           flush_ingest / kGB, interval_flush_ingest / kGB);
1139
  value->append(buf);
1140 1141
  snprintf(buf, sizeof(buf), "AddFile(GB): cumulative %.3f, interval %.3f\n",
           add_file_ingest / kGB, interval_add_file_inget / kGB);
1142 1143
  value->append(buf);

1144 1145
  uint64_t interval_ingest_files_addfile =
      ingest_files_addfile - cf_stats_snapshot_.ingest_files_addfile;
I
Islam AbdelRahman 已提交
1146 1147
  snprintf(buf, sizeof(buf), "AddFile(Total Files): cumulative %" PRIu64
                             ", interval %" PRIu64 "\n",
1148 1149 1150 1151 1152 1153
           ingest_files_addfile, interval_ingest_files_addfile);
  value->append(buf);

  uint64_t interval_ingest_l0_files_addfile =
      ingest_l0_files_addfile - cf_stats_snapshot_.ingest_l0_files_addfile;
  snprintf(buf, sizeof(buf),
I
Islam AbdelRahman 已提交
1154
           "AddFile(L0 Files): cumulative %" PRIu64 ", interval %" PRIu64 "\n",
1155 1156 1157 1158 1159
           ingest_l0_files_addfile, interval_ingest_l0_files_addfile);
  value->append(buf);

  uint64_t interval_ingest_keys_addfile =
      ingest_keys_addfile - cf_stats_snapshot_.ingest_keys_addfile;
I
Islam AbdelRahman 已提交
1160 1161
  snprintf(buf, sizeof(buf),
           "AddFile(Keys): cumulative %" PRIu64 ", interval %" PRIu64 "\n",
1162 1163 1164
           ingest_keys_addfile, interval_ingest_keys_addfile);
  value->append(buf);

1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205
  // Compact
  uint64_t compact_bytes_read = 0;
  uint64_t compact_bytes_write = 0;
  uint64_t compact_micros = 0;
  for (int level = 0; level < number_levels_; level++) {
    compact_bytes_read += comp_stats_[level].bytes_read_output_level +
                          comp_stats_[level].bytes_read_non_output_levels;
    compact_bytes_write += comp_stats_[level].bytes_written;
    compact_micros += comp_stats_[level].micros;
  }

  snprintf(buf, sizeof(buf),
           "Cumulative compaction: %.2f GB write, %.2f MB/s write, "
           "%.2f GB read, %.2f MB/s read, %.1f seconds\n",
           compact_bytes_write / kGB, compact_bytes_write / kMB / seconds_up,
           compact_bytes_read / kGB, compact_bytes_read / kMB / seconds_up,
           compact_micros / kMicrosInSec);
  value->append(buf);

  // Compaction interval
  uint64_t interval_compact_bytes_write =
      compact_bytes_write - cf_stats_snapshot_.compact_bytes_write;
  uint64_t interval_compact_bytes_read =
      compact_bytes_read - cf_stats_snapshot_.compact_bytes_read;
  uint64_t interval_compact_micros =
      compact_micros - cf_stats_snapshot_.compact_micros;

  snprintf(
      buf, sizeof(buf),
      "Interval compaction: %.2f GB write, %.2f MB/s write, "
      "%.2f GB read, %.2f MB/s read, %.1f seconds\n",
      interval_compact_bytes_write / kGB,
      interval_compact_bytes_write / kMB / std::max(interval_seconds_up, 0.001),
      interval_compact_bytes_read / kGB,
      interval_compact_bytes_read / kMB / std::max(interval_seconds_up, 0.001),
      interval_compact_micros / kMicrosInSec);
  value->append(buf);
  cf_stats_snapshot_.compact_bytes_write = compact_bytes_write;
  cf_stats_snapshot_.compact_bytes_read = compact_bytes_read;
  cf_stats_snapshot_.compact_micros = compact_micros;

1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231
  snprintf(buf, sizeof(buf),
           "Stalls(count): %" PRIu64
           " level0_slowdown, "
           "%" PRIu64
           " level0_slowdown_with_compaction, "
           "%" PRIu64
           " level0_numfiles, "
           "%" PRIu64
           " level0_numfiles_with_compaction, "
           "%" PRIu64
           " stop for pending_compaction_bytes, "
           "%" PRIu64
           " slowdown for pending_compaction_bytes, "
           "%" PRIu64
           " memtable_compaction, "
           "%" PRIu64
           " memtable_slowdown, "
           "interval %" PRIu64 " total count\n",
           cf_stats_count_[L0_FILE_COUNT_LIMIT_SLOWDOWNS],
           cf_stats_count_[LOCKED_L0_FILE_COUNT_LIMIT_SLOWDOWNS],
           cf_stats_count_[L0_FILE_COUNT_LIMIT_STOPS],
           cf_stats_count_[LOCKED_L0_FILE_COUNT_LIMIT_STOPS],
           cf_stats_count_[PENDING_COMPACTION_BYTES_LIMIT_STOPS],
           cf_stats_count_[PENDING_COMPACTION_BYTES_LIMIT_SLOWDOWNS],
           cf_stats_count_[MEMTABLE_LIMIT_STOPS],
           cf_stats_count_[MEMTABLE_LIMIT_SLOWDOWNS],
1232
           total_stall_count - cf_stats_snapshot_.stall_count);
1233 1234
  value->append(buf);

1235
  cf_stats_snapshot_.seconds_up = seconds_up;
1236
  cf_stats_snapshot_.ingest_bytes_flush = flush_ingest;
1237 1238 1239 1240
  cf_stats_snapshot_.ingest_bytes_addfile = add_file_ingest;
  cf_stats_snapshot_.ingest_files_addfile = ingest_files_addfile;
  cf_stats_snapshot_.ingest_l0_files_addfile = ingest_l0_files_addfile;
  cf_stats_snapshot_.ingest_keys_addfile = ingest_keys_addfile;
1241
  cf_stats_snapshot_.comp_stats = compaction_stats_sum;
1242 1243 1244
  cf_stats_snapshot_.stall_count = total_stall_count;
}

1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261
void InternalStats::DumpCFFileHistogram(std::string* value) {
  char buf[2000];
  snprintf(buf, sizeof(buf),
           "\n** File Read Latency Histogram By Level [%s] **\n",
           cfd_->GetName().c_str());
  value->append(buf);

  for (int level = 0; level < number_levels_; level++) {
    if (!file_read_latency_[level].Empty()) {
      char buf2[5000];
      snprintf(buf2, sizeof(buf2),
               "** Level %d read latency histogram (micros):\n%s\n", level,
               file_read_latency_[level].ToString().c_str());
      value->append(buf2);
    }
  }
}
1262 1263 1264

#else

1265
const DBPropertyInfo* GetPropertyInfo(const Slice& property) { return nullptr; }
1266 1267 1268

#endif  // !ROCKSDB_LITE

I
Igor Canadi 已提交
1269
}  // namespace rocksdb