internal_stats.cc 53.4 KB
Newer Older
S
Siying Dong 已提交
1 2 3
//  This source code is licensed under both the GPLv2 (found in the
//  COPYING file in the root directory) and Apache 2.0 License
//  (found in the LICENSE.Apache file in the root directory).
I
Igor Canadi 已提交
4 5 6 7 8 9
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.

#include "db/internal_stats.h"
L
liuhuahang 已提交
10 11

#ifndef __STDC_FORMAT_MACROS
I
Igor Canadi 已提交
12
#define __STDC_FORMAT_MACROS
L
liuhuahang 已提交
13 14
#endif

I
Igor Canadi 已提交
15
#include <inttypes.h>
16
#include <algorithm>
Y
Yi Wu 已提交
17 18
#include <limits>
#include <string>
19
#include <utility>
I
Igor Canadi 已提交
20
#include <vector>
21
#include "db/column_family.h"
22

23
#include "db/db_impl.h"
24
#include "util/string_util.h"
I
Igor Canadi 已提交
25 26 27

namespace rocksdb {

28
#ifndef ROCKSDB_LITE
29 30 31 32 33 34

const std::map<LevelStatType, LevelStat> InternalStats::compaction_level_stats =
    {
        {LevelStatType::NUM_FILES, LevelStat{"NumFiles", "Files"}},
        {LevelStatType::COMPACTED_FILES,
         LevelStat{"CompactedFiles", "CompactedFiles"}},
35
        {LevelStatType::SIZE_BYTES, LevelStat{"SizeBytes", "Size"}},
36
        {LevelStatType::SCORE, LevelStat{"Score", "Score"}},
37 38 39 40 41 42
        {LevelStatType::READ_GB, LevelStat{"ReadGB", "Read(GB)"}},
        {LevelStatType::RN_GB, LevelStat{"RnGB", "Rn(GB)"}},
        {LevelStatType::RNP1_GB, LevelStat{"Rnp1GB", "Rnp1(GB)"}},
        {LevelStatType::WRITE_GB, LevelStat{"WriteGB", "Write(GB)"}},
        {LevelStatType::W_NEW_GB, LevelStat{"WnewGB", "Wnew(GB)"}},
        {LevelStatType::MOVED_GB, LevelStat{"MovedGB", "Moved(GB)"}},
43
        {LevelStatType::WRITE_AMP, LevelStat{"WriteAmp", "W-Amp"}},
44 45 46 47 48
        {LevelStatType::READ_MBPS, LevelStat{"ReadMBps", "Rd(MB/s)"}},
        {LevelStatType::WRITE_MBPS, LevelStat{"WriteMBps", "Wr(MB/s)"}},
        {LevelStatType::COMP_SEC, LevelStat{"CompSec", "Comp(sec)"}},
        {LevelStatType::COMP_COUNT, LevelStat{"CompCount", "Comp(cnt)"}},
        {LevelStatType::AVG_SEC, LevelStat{"AvgSec", "Avg(sec)"}},
49 50 51 52
        {LevelStatType::KEY_IN, LevelStat{"KeyIn", "KeyIn"}},
        {LevelStatType::KEY_DROP, LevelStat{"KeyDrop", "KeyDrop"}},
};

L
Lei Jin 已提交
53 54 55
namespace {
const double kMB = 1048576.0;
const double kGB = kMB * 1024;
56
const double kMicrosInSec = 1000000.0;
L
Lei Jin 已提交
57

58
void PrintLevelStatsHeader(char* buf, size_t len, const std::string& cf_name) {
59 60 61 62 63 64 65
  int written_size =
      snprintf(buf, len, "\n** Compaction Stats [%s] **\n", cf_name.c_str());
  auto hdr = [](LevelStatType t) {
    return InternalStats::compaction_level_stats.at(t).header_name.c_str();
  };
  int line_size = snprintf(
      buf + written_size, len - written_size,
66
      "Level    %s   %s     %s %s  %s %s %s %s %s %s %s %s %s %s %s %s %s\n",
67
      // Note that we skip COMPACTED_FILES and merge it with Files column
68
      hdr(LevelStatType::NUM_FILES), hdr(LevelStatType::SIZE_BYTES),
69 70 71 72 73 74 75 76 77 78 79 80
      hdr(LevelStatType::SCORE), hdr(LevelStatType::READ_GB),
      hdr(LevelStatType::RN_GB), hdr(LevelStatType::RNP1_GB),
      hdr(LevelStatType::WRITE_GB), hdr(LevelStatType::W_NEW_GB),
      hdr(LevelStatType::MOVED_GB), hdr(LevelStatType::WRITE_AMP),
      hdr(LevelStatType::READ_MBPS), hdr(LevelStatType::WRITE_MBPS),
      hdr(LevelStatType::COMP_SEC), hdr(LevelStatType::COMP_COUNT),
      hdr(LevelStatType::AVG_SEC), hdr(LevelStatType::KEY_IN),
      hdr(LevelStatType::KEY_DROP));

  written_size += line_size;
  snprintf(buf + written_size, len - written_size, "%s\n",
           std::string(line_size, '-').c_str());
L
Lei Jin 已提交
81 82
}

83 84 85 86
void PrepareLevelStats(std::map<LevelStatType, double>* level_stats,
                       int num_files, int being_compacted,
                       double total_file_size, double score, double w_amp,
                       const InternalStats::CompactionStats& stats) {
87 88 89 90
  uint64_t bytes_read =
      stats.bytes_read_non_output_levels + stats.bytes_read_output_level;
  int64_t bytes_new =
      stats.bytes_written - stats.bytes_read_output_level;
91
  double elapsed = (stats.micros + 1) / kMicrosInSec;
L
Lei Jin 已提交
92

93 94
  (*level_stats)[LevelStatType::NUM_FILES] = num_files;
  (*level_stats)[LevelStatType::COMPACTED_FILES] = being_compacted;
95
  (*level_stats)[LevelStatType::SIZE_BYTES] = total_file_size;
96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119
  (*level_stats)[LevelStatType::SCORE] = score;
  (*level_stats)[LevelStatType::READ_GB] = bytes_read / kGB;
  (*level_stats)[LevelStatType::RN_GB] =
      stats.bytes_read_non_output_levels / kGB;
  (*level_stats)[LevelStatType::RNP1_GB] = stats.bytes_read_output_level / kGB;
  (*level_stats)[LevelStatType::WRITE_GB] = stats.bytes_written / kGB;
  (*level_stats)[LevelStatType::W_NEW_GB] = bytes_new / kGB;
  (*level_stats)[LevelStatType::MOVED_GB] = stats.bytes_moved / kGB;
  (*level_stats)[LevelStatType::WRITE_AMP] = w_amp;
  (*level_stats)[LevelStatType::READ_MBPS] = bytes_read / kMB / elapsed;
  (*level_stats)[LevelStatType::WRITE_MBPS] =
      stats.bytes_written / kMB / elapsed;
  (*level_stats)[LevelStatType::COMP_SEC] = stats.micros / kMicrosInSec;
  (*level_stats)[LevelStatType::COMP_COUNT] = stats.count;
  (*level_stats)[LevelStatType::AVG_SEC] =
      stats.count == 0 ? 0 : stats.micros / kMicrosInSec / stats.count;
  (*level_stats)[LevelStatType::KEY_IN] =
      static_cast<double>(stats.num_input_records);
  (*level_stats)[LevelStatType::KEY_DROP] =
      static_cast<double>(stats.num_dropped_records);
}

void PrintLevelStats(char* buf, size_t len, const std::string& name,
                     const std::map<LevelStatType, double>& stat_value) {
L
Lei Jin 已提交
120
  snprintf(buf, len,
121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138
           "%4s "      /*  Level */
           "%6d/%-3d " /*  Files */
           "%8s "      /*  Size */
           "%5.1f "    /*  Score */
           "%8.1f "    /*  Read(GB) */
           "%7.1f "    /*  Rn(GB) */
           "%8.1f "    /*  Rnp1(GB) */
           "%9.1f "    /*  Write(GB) */
           "%8.1f "    /*  Wnew(GB) */
           "%9.1f "    /*  Moved(GB) */
           "%5.1f "    /*  W-Amp */
           "%8.1f "    /*  Rd(MB/s) */
           "%8.1f "    /*  Wr(MB/s) */
           "%9.0f "    /*  Comp(sec) */
           "%9d "      /*  Comp(cnt) */
           "%8.3f "    /*  Avg(sec) */
           "%7s "      /*  KeyIn */
           "%6s\n",    /*  KeyDrop */
139
           name.c_str(),
140 141
           static_cast<int>(stat_value.at(LevelStatType::NUM_FILES)),
           static_cast<int>(stat_value.at(LevelStatType::COMPACTED_FILES)),
142 143 144
           BytesToHumanString(
               static_cast<uint64_t>(stat_value.at(LevelStatType::SIZE_BYTES)))
               .c_str(),
145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173
           stat_value.at(LevelStatType::SCORE),
           stat_value.at(LevelStatType::READ_GB),
           stat_value.at(LevelStatType::RN_GB),
           stat_value.at(LevelStatType::RNP1_GB),
           stat_value.at(LevelStatType::WRITE_GB),
           stat_value.at(LevelStatType::W_NEW_GB),
           stat_value.at(LevelStatType::MOVED_GB),
           stat_value.at(LevelStatType::WRITE_AMP),
           stat_value.at(LevelStatType::READ_MBPS),
           stat_value.at(LevelStatType::WRITE_MBPS),
           stat_value.at(LevelStatType::COMP_SEC),
           static_cast<int>(stat_value.at(LevelStatType::COMP_COUNT)),
           stat_value.at(LevelStatType::AVG_SEC),
           NumberToHumanString(
               static_cast<std::int64_t>(stat_value.at(LevelStatType::KEY_IN)))
               .c_str(),
           NumberToHumanString(static_cast<std::int64_t>(
                                   stat_value.at(LevelStatType::KEY_DROP)))
               .c_str());
}

void PrintLevelStats(char* buf, size_t len, const std::string& name,
                     int num_files, int being_compacted, double total_file_size,
                     double score, double w_amp,
                     const InternalStats::CompactionStats& stats) {
  std::map<LevelStatType, double> level_stats;
  PrepareLevelStats(&level_stats, num_files, being_compacted, total_file_size,
                    score, w_amp, stats);
  PrintLevelStats(buf, len, name, level_stats);
L
Lei Jin 已提交
174
}
175 176 177 178 179 180 181 182 183 184 185 186 187

// Assumes that trailing numbers represent an optional argument. This requires
// property names to not end with numbers.
std::pair<Slice, Slice> GetPropertyNameAndArg(const Slice& property) {
  Slice name = property, arg = property;
  size_t sfx_len = 0;
  while (sfx_len < property.size() &&
         isdigit(property[property.size() - sfx_len - 1])) {
    ++sfx_len;
  }
  name.remove_suffix(sfx_len);
  arg.remove_prefix(property.size() - sfx_len);
  return {name, arg};
L
Lei Jin 已提交
188
}
189
}  // anonymous namespace
L
Lei Jin 已提交
190

191 192 193
static const std::string rocksdb_prefix = "rocksdb.";

static const std::string num_files_at_level_prefix = "num-files-at-level";
194 195
static const std::string compression_ratio_at_level_prefix =
    "compression-ratio-at-level";
196
static const std::string allstats = "stats";
197 198
static const std::string sstables = "sstables";
static const std::string cfstats = "cfstats";
199 200 201
static const std::string cfstats_no_file_histogram =
    "cfstats-no-file-histogram";
static const std::string cf_file_histogram = "cf-file-histogram";
202 203 204
static const std::string dbstats = "dbstats";
static const std::string levelstats = "levelstats";
static const std::string num_immutable_mem_table = "num-immutable-mem-table";
205 206
static const std::string num_immutable_mem_table_flushed =
    "num-immutable-mem-table-flushed";
207 208 209 210 211
static const std::string mem_table_flush_pending = "mem-table-flush-pending";
static const std::string compaction_pending = "compaction-pending";
static const std::string background_errors = "background-errors";
static const std::string cur_size_active_mem_table =
                          "cur-size-active-mem-table";
212 213
static const std::string cur_size_all_mem_tables = "cur-size-all-mem-tables";
static const std::string size_all_mem_tables = "size-all-mem-tables";
214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229
static const std::string num_entries_active_mem_table =
                          "num-entries-active-mem-table";
static const std::string num_entries_imm_mem_tables =
                          "num-entries-imm-mem-tables";
static const std::string num_deletes_active_mem_table =
                          "num-deletes-active-mem-table";
static const std::string num_deletes_imm_mem_tables =
                          "num-deletes-imm-mem-tables";
static const std::string estimate_num_keys = "estimate-num-keys";
static const std::string estimate_table_readers_mem =
                          "estimate-table-readers-mem";
static const std::string is_file_deletions_enabled =
                          "is-file-deletions-enabled";
static const std::string num_snapshots = "num-snapshots";
static const std::string oldest_snapshot_time = "oldest-snapshot-time";
static const std::string num_live_versions = "num-live-versions";
230 231
static const std::string current_version_number =
    "current-super-version-number";
A
Andres Notzli 已提交
232
static const std::string estimate_live_data_size = "estimate-live-data-size";
233
static const std::string min_log_number_to_keep = "min-log-number-to-keep";
234
static const std::string base_level = "base-level";
235
static const std::string total_sst_files_size = "total-sst-files-size";
236
static const std::string live_sst_files_size = "live-sst-files-size";
237 238
static const std::string estimate_pending_comp_bytes =
    "estimate-pending-compaction-bytes";
239 240 241 242
static const std::string aggregated_table_properties =
    "aggregated-table-properties";
static const std::string aggregated_table_properties_at_level =
    aggregated_table_properties + "-at-level";
243 244
static const std::string num_running_compactions = "num-running-compactions";
static const std::string num_running_flushes = "num-running-flushes";
245 246 247
static const std::string actual_delayed_write_rate =
    "actual-delayed-write-rate";
static const std::string is_write_stopped = "is-write-stopped";
Y
Yi Wu 已提交
248
static const std::string estimate_oldest_key_time = "estimate-oldest-key-time";
249 250 251

const std::string DB::Properties::kNumFilesAtLevelPrefix =
                      rocksdb_prefix + num_files_at_level_prefix;
252 253
const std::string DB::Properties::kCompressionRatioAtLevelPrefix =
                      rocksdb_prefix + compression_ratio_at_level_prefix;
254
const std::string DB::Properties::kStats = rocksdb_prefix + allstats;
255 256
const std::string DB::Properties::kSSTables = rocksdb_prefix + sstables;
const std::string DB::Properties::kCFStats = rocksdb_prefix + cfstats;
257 258 259 260
const std::string DB::Properties::kCFStatsNoFileHistogram =
    rocksdb_prefix + cfstats_no_file_histogram;
const std::string DB::Properties::kCFFileHistogram =
    rocksdb_prefix + cf_file_histogram;
261
const std::string DB::Properties::kDBStats = rocksdb_prefix + dbstats;
262
const std::string DB::Properties::kLevelStats = rocksdb_prefix + levelstats;
263 264
const std::string DB::Properties::kNumImmutableMemTable =
                      rocksdb_prefix + num_immutable_mem_table;
265
const std::string DB::Properties::kNumImmutableMemTableFlushed =
266
    rocksdb_prefix + num_immutable_mem_table_flushed;
267 268 269 270
const std::string DB::Properties::kMemTableFlushPending =
                      rocksdb_prefix + mem_table_flush_pending;
const std::string DB::Properties::kCompactionPending =
                      rocksdb_prefix + compaction_pending;
271 272 273 274
const std::string DB::Properties::kNumRunningCompactions =
    rocksdb_prefix + num_running_compactions;
const std::string DB::Properties::kNumRunningFlushes =
    rocksdb_prefix + num_running_flushes;
275 276 277 278 279
const std::string DB::Properties::kBackgroundErrors =
                      rocksdb_prefix + background_errors;
const std::string DB::Properties::kCurSizeActiveMemTable =
                      rocksdb_prefix + cur_size_active_mem_table;
const std::string DB::Properties::kCurSizeAllMemTables =
280
    rocksdb_prefix + cur_size_all_mem_tables;
281 282
const std::string DB::Properties::kSizeAllMemTables =
    rocksdb_prefix + size_all_mem_tables;
283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302
const std::string DB::Properties::kNumEntriesActiveMemTable =
                      rocksdb_prefix + num_entries_active_mem_table;
const std::string DB::Properties::kNumEntriesImmMemTables =
                      rocksdb_prefix + num_entries_imm_mem_tables;
const std::string DB::Properties::kNumDeletesActiveMemTable =
                      rocksdb_prefix + num_deletes_active_mem_table;
const std::string DB::Properties::kNumDeletesImmMemTables =
                      rocksdb_prefix + num_deletes_imm_mem_tables;
const std::string DB::Properties::kEstimateNumKeys =
                      rocksdb_prefix + estimate_num_keys;
const std::string DB::Properties::kEstimateTableReadersMem =
                      rocksdb_prefix + estimate_table_readers_mem;
const std::string DB::Properties::kIsFileDeletionsEnabled =
                      rocksdb_prefix + is_file_deletions_enabled;
const std::string DB::Properties::kNumSnapshots =
                      rocksdb_prefix + num_snapshots;
const std::string DB::Properties::kOldestSnapshotTime =
                      rocksdb_prefix + oldest_snapshot_time;
const std::string DB::Properties::kNumLiveVersions =
                      rocksdb_prefix + num_live_versions;
303
const std::string DB::Properties::kCurrentSuperVersionNumber =
304
    rocksdb_prefix + current_version_number;
A
Andres Notzli 已提交
305 306
const std::string DB::Properties::kEstimateLiveDataSize =
                      rocksdb_prefix + estimate_live_data_size;
307 308
const std::string DB::Properties::kMinLogNumberToKeep =
    rocksdb_prefix + min_log_number_to_keep;
309 310
const std::string DB::Properties::kTotalSstFilesSize =
                      rocksdb_prefix + total_sst_files_size;
311 312
const std::string DB::Properties::kLiveSstFilesSize =
    rocksdb_prefix + live_sst_files_size;
313
const std::string DB::Properties::kBaseLevel = rocksdb_prefix + base_level;
314 315
const std::string DB::Properties::kEstimatePendingCompactionBytes =
    rocksdb_prefix + estimate_pending_comp_bytes;
316 317 318 319
const std::string DB::Properties::kAggregatedTableProperties =
    rocksdb_prefix + aggregated_table_properties;
const std::string DB::Properties::kAggregatedTablePropertiesAtLevel =
    rocksdb_prefix + aggregated_table_properties_at_level;
320 321 322 323
const std::string DB::Properties::kActualDelayedWriteRate =
    rocksdb_prefix + actual_delayed_write_rate;
const std::string DB::Properties::kIsWriteStopped =
    rocksdb_prefix + is_write_stopped;
Y
Yi Wu 已提交
324 325
const std::string DB::Properties::kEstimateOldestKeyTime =
    rocksdb_prefix + estimate_oldest_key_time;
326

327 328 329 330 331 332 333 334 335 336 337 338 339 340
const std::unordered_map<std::string, DBPropertyInfo>
    InternalStats::ppt_name_to_info = {
        {DB::Properties::kNumFilesAtLevelPrefix,
         {false, &InternalStats::HandleNumFilesAtLevel, nullptr, nullptr}},
        {DB::Properties::kCompressionRatioAtLevelPrefix,
         {false, &InternalStats::HandleCompressionRatioAtLevelPrefix, nullptr,
          nullptr}},
        {DB::Properties::kLevelStats,
         {false, &InternalStats::HandleLevelStats, nullptr, nullptr}},
        {DB::Properties::kStats,
         {false, &InternalStats::HandleStats, nullptr, nullptr}},
        {DB::Properties::kCFStats,
         {false, &InternalStats::HandleCFStats, nullptr,
          &InternalStats::HandleCFMapStats}},
341 342 343 344 345
        {DB::Properties::kCFStatsNoFileHistogram,
         {false, &InternalStats::HandleCFStatsNoFileHistogram, nullptr,
          nullptr}},
        {DB::Properties::kCFFileHistogram,
         {false, &InternalStats::HandleCFFileHistogram, nullptr, nullptr}},
346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404
        {DB::Properties::kDBStats,
         {false, &InternalStats::HandleDBStats, nullptr, nullptr}},
        {DB::Properties::kSSTables,
         {false, &InternalStats::HandleSsTables, nullptr, nullptr}},
        {DB::Properties::kAggregatedTableProperties,
         {false, &InternalStats::HandleAggregatedTableProperties, nullptr,
          nullptr}},
        {DB::Properties::kAggregatedTablePropertiesAtLevel,
         {false, &InternalStats::HandleAggregatedTablePropertiesAtLevel,
          nullptr, nullptr}},
        {DB::Properties::kNumImmutableMemTable,
         {false, nullptr, &InternalStats::HandleNumImmutableMemTable, nullptr}},
        {DB::Properties::kNumImmutableMemTableFlushed,
         {false, nullptr, &InternalStats::HandleNumImmutableMemTableFlushed,
          nullptr}},
        {DB::Properties::kMemTableFlushPending,
         {false, nullptr, &InternalStats::HandleMemTableFlushPending, nullptr}},
        {DB::Properties::kCompactionPending,
         {false, nullptr, &InternalStats::HandleCompactionPending, nullptr}},
        {DB::Properties::kBackgroundErrors,
         {false, nullptr, &InternalStats::HandleBackgroundErrors, nullptr}},
        {DB::Properties::kCurSizeActiveMemTable,
         {false, nullptr, &InternalStats::HandleCurSizeActiveMemTable,
          nullptr}},
        {DB::Properties::kCurSizeAllMemTables,
         {false, nullptr, &InternalStats::HandleCurSizeAllMemTables, nullptr}},
        {DB::Properties::kSizeAllMemTables,
         {false, nullptr, &InternalStats::HandleSizeAllMemTables, nullptr}},
        {DB::Properties::kNumEntriesActiveMemTable,
         {false, nullptr, &InternalStats::HandleNumEntriesActiveMemTable,
          nullptr}},
        {DB::Properties::kNumEntriesImmMemTables,
         {false, nullptr, &InternalStats::HandleNumEntriesImmMemTables,
          nullptr}},
        {DB::Properties::kNumDeletesActiveMemTable,
         {false, nullptr, &InternalStats::HandleNumDeletesActiveMemTable,
          nullptr}},
        {DB::Properties::kNumDeletesImmMemTables,
         {false, nullptr, &InternalStats::HandleNumDeletesImmMemTables,
          nullptr}},
        {DB::Properties::kEstimateNumKeys,
         {false, nullptr, &InternalStats::HandleEstimateNumKeys, nullptr}},
        {DB::Properties::kEstimateTableReadersMem,
         {true, nullptr, &InternalStats::HandleEstimateTableReadersMem,
          nullptr}},
        {DB::Properties::kIsFileDeletionsEnabled,
         {false, nullptr, &InternalStats::HandleIsFileDeletionsEnabled,
          nullptr}},
        {DB::Properties::kNumSnapshots,
         {false, nullptr, &InternalStats::HandleNumSnapshots, nullptr}},
        {DB::Properties::kOldestSnapshotTime,
         {false, nullptr, &InternalStats::HandleOldestSnapshotTime, nullptr}},
        {DB::Properties::kNumLiveVersions,
         {false, nullptr, &InternalStats::HandleNumLiveVersions, nullptr}},
        {DB::Properties::kCurrentSuperVersionNumber,
         {false, nullptr, &InternalStats::HandleCurrentSuperVersionNumber,
          nullptr}},
        {DB::Properties::kEstimateLiveDataSize,
         {true, nullptr, &InternalStats::HandleEstimateLiveDataSize, nullptr}},
405 406
        {DB::Properties::kMinLogNumberToKeep,
         {false, nullptr, &InternalStats::HandleMinLogNumberToKeep, nullptr}},
407 408 409 410
        {DB::Properties::kBaseLevel,
         {false, nullptr, &InternalStats::HandleBaseLevel, nullptr}},
        {DB::Properties::kTotalSstFilesSize,
         {false, nullptr, &InternalStats::HandleTotalSstFilesSize, nullptr}},
411 412
        {DB::Properties::kLiveSstFilesSize,
         {false, nullptr, &InternalStats::HandleLiveSstFilesSize, nullptr}},
413 414 415 416 417 418 419 420
        {DB::Properties::kEstimatePendingCompactionBytes,
         {false, nullptr, &InternalStats::HandleEstimatePendingCompactionBytes,
          nullptr}},
        {DB::Properties::kNumRunningFlushes,
         {false, nullptr, &InternalStats::HandleNumRunningFlushes, nullptr}},
        {DB::Properties::kNumRunningCompactions,
         {false, nullptr, &InternalStats::HandleNumRunningCompactions,
          nullptr}},
421 422 423 424 425
        {DB::Properties::kActualDelayedWriteRate,
         {false, nullptr, &InternalStats::HandleActualDelayedWriteRate,
          nullptr}},
        {DB::Properties::kIsWriteStopped,
         {false, nullptr, &InternalStats::HandleIsWriteStopped, nullptr}},
Y
Yi Wu 已提交
426 427 428
        {DB::Properties::kEstimateOldestKeyTime,
         {false, nullptr, &InternalStats::HandleEstimateOldestKeyTime,
          nullptr}},
429 430 431 432 433 434 435
};

const DBPropertyInfo* GetPropertyInfo(const Slice& property) {
  std::string ppt_name = GetPropertyNameAndArg(property).first.ToString();
  auto ppt_info_iter = InternalStats::ppt_name_to_info.find(ppt_name);
  if (ppt_info_iter == InternalStats::ppt_name_to_info.end()) {
    return nullptr;
436
  }
437 438
  return &ppt_info_iter->second;
}
439

440 441 442 443 444 445 446
bool InternalStats::GetStringProperty(const DBPropertyInfo& property_info,
                                      const Slice& property,
                                      std::string* value) {
  assert(value != nullptr);
  assert(property_info.handle_string != nullptr);
  Slice arg = GetPropertyNameAndArg(property).second;
  return (this->*(property_info.handle_string))(value, arg);
447 448
}

449
bool InternalStats::GetMapProperty(const DBPropertyInfo& property_info,
450
                                   const Slice& property,
451
                                   std::map<std::string, std::string>* value) {
452 453 454 455 456
  assert(value != nullptr);
  assert(property_info.handle_map != nullptr);
  return (this->*(property_info.handle_map))(value);
}

457 458
bool InternalStats::GetIntProperty(const DBPropertyInfo& property_info,
                                   uint64_t* value, DBImpl* db) {
459
  assert(value != nullptr);
460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475
  assert(property_info.handle_int != nullptr &&
         !property_info.need_out_of_mutex);
  db->mutex_.AssertHeld();
  return (this->*(property_info.handle_int))(value, db, nullptr /* version */);
}

bool InternalStats::GetIntPropertyOutOfMutex(
    const DBPropertyInfo& property_info, Version* version, uint64_t* value) {
  assert(value != nullptr);
  assert(property_info.handle_int != nullptr &&
         property_info.need_out_of_mutex);
  return (this->*(property_info.handle_int))(value, nullptr /* db */, version);
}

bool InternalStats::HandleNumFilesAtLevel(std::string* value, Slice suffix) {
  uint64_t level;
A
Andres Notzli 已提交
476
  const auto* vstorage = cfd_->current()->storage_info();
477 478 479 480 481 482 483 484 485 486 487 488
  bool ok = ConsumeDecimalNumber(&suffix, &level) && suffix.empty();
  if (!ok || static_cast<int>(level) >= number_levels_) {
    return false;
  } else {
    char buf[100];
    snprintf(buf, sizeof(buf), "%d",
             vstorage->NumLevelFiles(static_cast<int>(level)));
    *value = buf;
    return true;
  }
}

489 490 491 492 493 494 495 496 497 498 499 500 501
bool InternalStats::HandleCompressionRatioAtLevelPrefix(std::string* value,
                                                        Slice suffix) {
  uint64_t level;
  const auto* vstorage = cfd_->current()->storage_info();
  bool ok = ConsumeDecimalNumber(&suffix, &level) && suffix.empty();
  if (!ok || level >= static_cast<uint64_t>(number_levels_)) {
    return false;
  }
  *value = ToString(
      vstorage->GetEstimatedCompressionRatioAtLevel(static_cast<int>(level)));
  return true;
}

502
bool InternalStats::HandleLevelStats(std::string* value, Slice suffix) {
503 504 505 506 507 508
  char buf[1000];
  const auto* vstorage = cfd_->current()->storage_info();
  snprintf(buf, sizeof(buf),
           "Level Files Size(MB)\n"
           "--------------------\n");
  value->append(buf);
A
Andres Notzli 已提交
509

510 511 512 513 514
  for (int level = 0; level < number_levels_; level++) {
    snprintf(buf, sizeof(buf), "%3d %8d %8.0f\n", level,
             vstorage->NumLevelFiles(level),
             vstorage->NumLevelBytes(level) / kMB);
    value->append(buf);
515
  }
516
  return true;
517
}
518

519 520 521 522 523 524 525 526 527 528
bool InternalStats::HandleStats(std::string* value, Slice suffix) {
  if (!HandleCFStats(value, suffix)) {
    return false;
  }
  if (!HandleDBStats(value, suffix)) {
    return false;
  }
  return true;
}

529 530
bool InternalStats::HandleCFMapStats(
    std::map<std::string, std::string>* cf_stats) {
531 532 533 534
  DumpCFMapStats(cf_stats);
  return true;
}

535
bool InternalStats::HandleCFStats(std::string* value, Slice suffix) {
536 537 538 539
  DumpCFStats(value);
  return true;
}

540
bool InternalStats::HandleCFStatsNoFileHistogram(std::string* value,
541
                                                 Slice suffix) {
542 543 544 545
  DumpCFStatsNoFileHistogram(value);
  return true;
}

546
bool InternalStats::HandleCFFileHistogram(std::string* value, Slice suffix) {
547 548 549 550
  DumpCFFileHistogram(value);
  return true;
}

551
bool InternalStats::HandleDBStats(std::string* value, Slice suffix) {
552 553 554 555
  DumpDBStats(value);
  return true;
}

556
bool InternalStats::HandleSsTables(std::string* value, Slice suffix) {
S
sdong 已提交
557
  auto* current = cfd_->current();
558
  *value = current->DebugString(true, true);
559 560
  return true;
}
I
Igor Canadi 已提交
561

562
bool InternalStats::HandleAggregatedTableProperties(std::string* value,
563
                                                    Slice suffix) {
564 565 566 567
  std::shared_ptr<const TableProperties> tp;
  auto s = cfd_->current()->GetAggregatedTableProperties(&tp);
  if (!s.ok()) {
    return false;
568
  }
569 570
  *value = tp->ToString();
  return true;
571 572
}

573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589
bool InternalStats::HandleAggregatedTablePropertiesAtLevel(std::string* value,
                                                           Slice suffix) {
  uint64_t level;
  bool ok = ConsumeDecimalNumber(&suffix, &level) && suffix.empty();
  if (!ok || static_cast<int>(level) >= number_levels_) {
    return false;
  }
  std::shared_ptr<const TableProperties> tp;
  auto s = cfd_->current()->GetAggregatedTableProperties(
      &tp, static_cast<int>(level));
  if (!s.ok()) {
    return false;
  }
  *value = tp->ToString();
  return true;
}

590 591
bool InternalStats::HandleNumImmutableMemTable(uint64_t* value, DBImpl* db,
                                               Version* version) {
592 593 594 595 596
  *value = cfd_->imm()->NumNotFlushed();
  return true;
}

bool InternalStats::HandleNumImmutableMemTableFlushed(uint64_t* value,
597 598
                                                      DBImpl* db,
                                                      Version* version) {
599 600 601 602
  *value = cfd_->imm()->NumFlushed();
  return true;
}

603 604
bool InternalStats::HandleMemTableFlushPending(uint64_t* value, DBImpl* db,
                                               Version* version) {
605 606 607 608 609 610
  // Return number of mem tables that are ready to flush (made immutable)
  *value = (cfd_->imm()->IsFlushPending() ? 1 : 0);
  return true;
}

bool InternalStats::HandleNumRunningFlushes(uint64_t* value, DBImpl* db,
611
                                            Version* version) {
612 613 614 615
  *value = db->num_running_flushes();
  return true;
}

616 617
bool InternalStats::HandleCompactionPending(uint64_t* value, DBImpl* db,
                                            Version* version) {
618 619
  // 1 if the system already determines at least one compaction is needed.
  // 0 otherwise,
S
sdong 已提交
620
  const auto* vstorage = cfd_->current()->storage_info();
621 622 623
  *value = (cfd_->compaction_picker()->NeedsCompaction(vstorage) ? 1 : 0);
  return true;
}
624

625
bool InternalStats::HandleNumRunningCompactions(uint64_t* value, DBImpl* db,
626
                                                Version* version) {
627 628 629 630
  *value = db->num_running_compactions_;
  return true;
}

631 632
bool InternalStats::HandleBackgroundErrors(uint64_t* value, DBImpl* db,
                                           Version* version) {
633 634 635 636 637
  // Accumulated number of  errors in background flushes or compactions.
  *value = GetBackgroundErrorCount();
  return true;
}

638 639
bool InternalStats::HandleCurSizeActiveMemTable(uint64_t* value, DBImpl* db,
                                                Version* version) {
640 641 642 643 644
  // Current size of the active memtable
  *value = cfd_->mem()->ApproximateMemoryUsage();
  return true;
}

645 646
bool InternalStats::HandleCurSizeAllMemTables(uint64_t* value, DBImpl* db,
                                              Version* version) {
647 648 649 650 651 652
  // Current size of the active memtable + immutable memtables
  *value = cfd_->mem()->ApproximateMemoryUsage() +
           cfd_->imm()->ApproximateUnflushedMemTablesMemoryUsage();
  return true;
}

653 654
bool InternalStats::HandleSizeAllMemTables(uint64_t* value, DBImpl* db,
                                           Version* version) {
655 656 657 658 659
  *value = cfd_->mem()->ApproximateMemoryUsage() +
           cfd_->imm()->ApproximateMemoryUsage();
  return true;
}

660 661
bool InternalStats::HandleNumEntriesActiveMemTable(uint64_t* value, DBImpl* db,
                                                   Version* version) {
662 663 664 665 666
  // Current number of entires in the active memtable
  *value = cfd_->mem()->num_entries();
  return true;
}

667 668
bool InternalStats::HandleNumEntriesImmMemTables(uint64_t* value, DBImpl* db,
                                                 Version* version) {
669 670 671 672 673
  // Current number of entries in the immutable memtables
  *value = cfd_->imm()->current()->GetTotalNumEntries();
  return true;
}

674 675
bool InternalStats::HandleNumDeletesActiveMemTable(uint64_t* value, DBImpl* db,
                                                   Version* version) {
676 677 678 679 680
  // Current number of entires in the active memtable
  *value = cfd_->mem()->num_deletes();
  return true;
}

681 682
bool InternalStats::HandleNumDeletesImmMemTables(uint64_t* value, DBImpl* db,
                                                 Version* version) {
683 684 685 686 687
  // Current number of entries in the immutable memtables
  *value = cfd_->imm()->current()->GetTotalNumDeletes();
  return true;
}

688 689
bool InternalStats::HandleEstimateNumKeys(uint64_t* value, DBImpl* db,
                                          Version* version) {
690 691 692
  // Estimate number of entries in the column family:
  // Use estimated entries in tables + total entries in memtables.
  const auto* vstorage = cfd_->current()->storage_info();
693 694 695 696 697 698 699 700
  uint64_t estimate_keys = cfd_->mem()->num_entries() +
                           cfd_->imm()->current()->GetTotalNumEntries() +
                           vstorage->GetEstimatedActiveKeys();
  uint64_t estimate_deletes =
      cfd_->mem()->num_deletes() + cfd_->imm()->current()->GetTotalNumDeletes();
  *value = estimate_keys > estimate_deletes * 2
               ? estimate_keys - (estimate_deletes * 2)
               : 0;
701 702 703 704
  return true;
}

bool InternalStats::HandleNumSnapshots(uint64_t* value, DBImpl* db,
705
                                       Version* version) {
706 707 708 709 710
  *value = db->snapshots().count();
  return true;
}

bool InternalStats::HandleOldestSnapshotTime(uint64_t* value, DBImpl* db,
711
                                             Version* version) {
712 713 714 715
  *value = static_cast<uint64_t>(db->snapshots().GetOldestSnapshotTime());
  return true;
}

716 717
bool InternalStats::HandleNumLiveVersions(uint64_t* value, DBImpl* db,
                                          Version* version) {
718 719 720 721
  *value = cfd_->GetNumLiveVersions();
  return true;
}

722 723
bool InternalStats::HandleCurrentSuperVersionNumber(uint64_t* value, DBImpl* db,
                                                    Version* version) {
724 725 726 727
  *value = cfd_->GetSuperVersionNumber();
  return true;
}

728
bool InternalStats::HandleIsFileDeletionsEnabled(uint64_t* value, DBImpl* db,
729
                                                 Version* version) {
730 731 732 733
  *value = db->IsFileDeletionsEnabled();
  return true;
}

734 735
bool InternalStats::HandleBaseLevel(uint64_t* value, DBImpl* db,
                                    Version* version) {
736 737 738 739 740
  const auto* vstorage = cfd_->current()->storage_info();
  *value = vstorage->base_level();
  return true;
}

741 742
bool InternalStats::HandleTotalSstFilesSize(uint64_t* value, DBImpl* /*db*/,
                                            Version* /*version*/) {
743 744 745 746
  *value = cfd_->GetTotalSstFilesSize();
  return true;
}

747 748 749 750 751 752
bool InternalStats::HandleLiveSstFilesSize(uint64_t* value, DBImpl* /*db*/,
                                           Version* /*version*/) {
  *value = cfd_->GetLiveSstFilesSize();
  return true;
}

753
bool InternalStats::HandleEstimatePendingCompactionBytes(uint64_t* value,
754 755
                                                         DBImpl* db,
                                                         Version* version) {
756 757 758 759 760
  const auto* vstorage = cfd_->current()->storage_info();
  *value = vstorage->estimated_compaction_needed_bytes();
  return true;
}

761
bool InternalStats::HandleEstimateTableReadersMem(uint64_t* value, DBImpl* db,
762 763 764 765 766
                                                  Version* version) {
  *value = (version == nullptr) ? 0 : version->GetMemoryUsageByTableReaders();
  return true;
}

767 768
bool InternalStats::HandleEstimateLiveDataSize(uint64_t* value, DBImpl* db,
                                               Version* version) {
769 770 771
  const auto* vstorage = cfd_->current()->storage_info();
  *value = vstorage->EstimateLiveDataSize();
  return true;
I
Igor Canadi 已提交
772 773
}

774
bool InternalStats::HandleMinLogNumberToKeep(uint64_t* value, DBImpl* db,
775
                                             Version* version) {
776 777 778 779
  *value = db->MinLogNumberToKeep();
  return true;
}

780
bool InternalStats::HandleActualDelayedWriteRate(uint64_t* value, DBImpl* db,
781
                                                 Version* version) {
782 783 784 785 786 787 788 789 790 791
  const WriteController& wc = db->write_controller();
  if (!wc.NeedsDelay()) {
    *value = 0;
  } else {
    *value = wc.delayed_write_rate();
  }
  return true;
}

bool InternalStats::HandleIsWriteStopped(uint64_t* value, DBImpl* db,
792
                                         Version* version) {
793 794 795 796
  *value = db->write_controller().IsStopped() ? 1 : 0;
  return true;
}

Y
Yi Wu 已提交
797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815
bool InternalStats::HandleEstimateOldestKeyTime(uint64_t* value, DBImpl* /*db*/,
                                                Version* /*version*/) {
  // TODO(yiwu): The property is currently available for fifo compaction
  // with allow_compaction = false. This is because we don't propagate
  // oldest_key_time on compaction.
  if (cfd_->ioptions()->compaction_style != kCompactionStyleFIFO ||
      cfd_->GetCurrentMutableCFOptions()
          ->compaction_options_fifo.allow_compaction) {
    return false;
  }

  TablePropertiesCollection collection;
  auto s = cfd_->current()->GetPropertiesOfAllTables(&collection);
  if (!s.ok()) {
    return false;
  }
  *value = std::numeric_limits<uint64_t>::max();
  for (auto& p : collection) {
    *value = std::min(*value, p.second->oldest_key_time);
816 817 818
    if (*value == 0) {
      break;
    }
Y
Yi Wu 已提交
819
  }
820 821 822 823 824
  if (*value > 0) {
    *value = std::min({cfd_->mem()->ApproximateOldestKeyTime(),
                       cfd_->imm()->ApproximateOldestKeyTime(), *value});
  }
  return *value > 0 && *value < std::numeric_limits<uint64_t>::max();
Y
Yi Wu 已提交
825 826
}

827 828 829
void InternalStats::DumpDBStats(std::string* value) {
  char buf[1000];
  // DB-level stats, only available from default column family
830
  double seconds_up = (env_->NowMicros() - started_at_ + 1) / kMicrosInSec;
831 832 833 834 835 836
  double interval_seconds_up = seconds_up - db_stats_snapshot_.seconds_up;
  snprintf(buf, sizeof(buf),
           "\n** DB Stats **\nUptime(secs): %.1f total, %.1f interval\n",
           seconds_up, interval_seconds_up);
  value->append(buf);
  // Cumulative
837 838 839 840 841 842 843 844
  uint64_t user_bytes_written = GetDBStats(InternalStats::BYTES_WRITTEN);
  uint64_t num_keys_written = GetDBStats(InternalStats::NUMBER_KEYS_WRITTEN);
  uint64_t write_other = GetDBStats(InternalStats::WRITE_DONE_BY_OTHER);
  uint64_t write_self = GetDBStats(InternalStats::WRITE_DONE_BY_SELF);
  uint64_t wal_bytes = GetDBStats(InternalStats::WAL_FILE_BYTES);
  uint64_t wal_synced = GetDBStats(InternalStats::WAL_FILE_SYNCED);
  uint64_t write_with_wal = GetDBStats(InternalStats::WRITE_WITH_WAL);
  uint64_t write_stall_micros = GetDBStats(InternalStats::WRITE_STALL_MICROS);
845

846 847 848
  const int kHumanMicrosLen = 32;
  char human_micros[kHumanMicrosLen];

849
  // Data
S
sdong 已提交
850 851
  // writes: total number of write requests.
  // keys: total number of key updates issued by all the write requests
852 853
  // commit groups: number of group commits issued to the DB. Each group can
  //                contain one or more writes.
S
sdong 已提交
854
  // so writes/keys is the average number of put in multi-put or put
855
  // writes/groups is the average group commit size.
S
sdong 已提交
856 857
  //
  // The format is the same for interval stats.
858
  snprintf(buf, sizeof(buf),
859 860
           "Cumulative writes: %s writes, %s keys, %s commit groups, "
           "%.1f writes per commit group, ingest: %.2f GB, %.2f MB/s\n",
861 862 863
           NumberToHumanString(write_other + write_self).c_str(),
           NumberToHumanString(num_keys_written).c_str(),
           NumberToHumanString(write_self).c_str(),
864
           (write_other + write_self) / static_cast<double>(write_self + 1),
865
           user_bytes_written / kGB, user_bytes_written / kMB / seconds_up);
866 867 868
  value->append(buf);
  // WAL
  snprintf(buf, sizeof(buf),
869 870 871 872
           "Cumulative WAL: %s writes, %s syncs, "
           "%.2f writes per sync, written: %.2f GB, %.2f MB/s\n",
           NumberToHumanString(write_with_wal).c_str(),
           NumberToHumanString(wal_synced).c_str(),
873
           write_with_wal / static_cast<double>(wal_synced + 1),
874 875 876 877 878 879 880 881 882
           wal_bytes / kGB, wal_bytes / kMB / seconds_up);
  value->append(buf);
  // Stall
  AppendHumanMicros(write_stall_micros, human_micros, kHumanMicrosLen, true);
  snprintf(buf, sizeof(buf),
           "Cumulative stall: %s, %.1f percent\n",
           human_micros,
           // 10000 = divide by 1M to get secs, then multiply by 100 for pct
           write_stall_micros / 10000.0 / std::max(seconds_up, 0.001));
883 884 885 886 887
  value->append(buf);

  // Interval
  uint64_t interval_write_other = write_other - db_stats_snapshot_.write_other;
  uint64_t interval_write_self = write_self - db_stats_snapshot_.write_self;
S
sdong 已提交
888 889
  uint64_t interval_num_keys_written =
      num_keys_written - db_stats_snapshot_.num_keys_written;
890
  snprintf(buf, sizeof(buf),
891 892
           "Interval writes: %s writes, %s keys, %s commit groups, "
           "%.1f writes per commit group, ingest: %.2f MB, %.2f MB/s\n",
893 894 895 896
           NumberToHumanString(
               interval_write_other + interval_write_self).c_str(),
           NumberToHumanString(interval_num_keys_written).c_str(),
           NumberToHumanString(interval_write_self).c_str(),
897 898
           static_cast<double>(interval_write_other + interval_write_self) /
               (interval_write_self + 1),
S
sdong 已提交
899
           (user_bytes_written - db_stats_snapshot_.ingest_bytes) / kMB,
900 901
           (user_bytes_written - db_stats_snapshot_.ingest_bytes) / kMB /
               std::max(interval_seconds_up, 0.001)),
902 903 904 905 906 907 908 909
  value->append(buf);

  uint64_t interval_write_with_wal =
      write_with_wal - db_stats_snapshot_.write_with_wal;
  uint64_t interval_wal_synced = wal_synced - db_stats_snapshot_.wal_synced;
  uint64_t interval_wal_bytes = wal_bytes - db_stats_snapshot_.wal_bytes;

  snprintf(buf, sizeof(buf),
910 911 912 913
           "Interval WAL: %s writes, %s syncs, "
           "%.2f writes per sync, written: %.2f MB, %.2f MB/s\n",
           NumberToHumanString(interval_write_with_wal).c_str(),
           NumberToHumanString(interval_wal_synced).c_str(),
914 915
           interval_write_with_wal /
              static_cast<double>(interval_wal_synced + 1),
916 917 918 919 920 921 922 923 924 925 926 927 928 929
           interval_wal_bytes / kGB,
           interval_wal_bytes / kMB / std::max(interval_seconds_up, 0.001));
  value->append(buf);

  // Stall
  AppendHumanMicros(
      write_stall_micros - db_stats_snapshot_.write_stall_micros,
      human_micros, kHumanMicrosLen, true);
  snprintf(buf, sizeof(buf),
           "Interval stall: %s, %.1f percent\n",
           human_micros,
           // 10000 = divide by 1M to get secs, then multiply by 100 for pct
           (write_stall_micros - db_stats_snapshot_.write_stall_micros) /
               10000.0 / std::max(interval_seconds_up, 0.001));
930 931 932 933 934 935
  value->append(buf);

  db_stats_snapshot_.seconds_up = seconds_up;
  db_stats_snapshot_.ingest_bytes = user_bytes_written;
  db_stats_snapshot_.write_other = write_other;
  db_stats_snapshot_.write_self = write_self;
S
sdong 已提交
936
  db_stats_snapshot_.num_keys_written = num_keys_written;
937 938 939
  db_stats_snapshot_.wal_bytes = wal_bytes;
  db_stats_snapshot_.wal_synced = wal_synced;
  db_stats_snapshot_.write_with_wal = write_with_wal;
S
sdong 已提交
940
  db_stats_snapshot_.write_stall_micros = write_stall_micros;
941 942
}

943
/**
944 945 946 947 948 949
 * Dump Compaction Level stats to a map of stat name with "compaction." prefix
 * to value in double as string. The level in stat name is represented with
 * a prefix "Lx" where "x" is the level number. A special level "Sum"
 * represents the sum of a stat for all levels.
 * The result also contains IO stall counters which keys start with "io_stalls."
 * and values represent uint64 encoded as strings.
950
 */
951 952
void InternalStats::DumpCFMapStats(
        std::map<std::string, std::string>* cf_stats) {
953 954 955 956 957 958 959 960 961
  CompactionStats compaction_stats_sum(0);
  std::map<int, std::map<LevelStatType, double>> levels_stats;
  DumpCFMapStats(&levels_stats, &compaction_stats_sum);
  for (auto const& level_ent : levels_stats) {
    auto level_str =
        level_ent.first == -1 ? "Sum" : "L" + ToString(level_ent.first);
    for (auto const& stat_ent : level_ent.second) {
      auto stat_type = stat_ent.first;
      auto key_str =
962
          "compaction." + level_str + "." +
963
          InternalStats::compaction_level_stats.at(stat_type).property_name;
964
      (*cf_stats)[key_str] = std::to_string(stat_ent.second);
965 966
    }
  }
967 968

  DumpCFMapStatsIOStalls(cf_stats);
969 970
}

971
void InternalStats::DumpCFMapStats(
972 973
    std::map<int, std::map<LevelStatType, double>>* levels_stats,
    CompactionStats* compaction_stats_sum) {
S
sdong 已提交
974
  const VersionStorageInfo* vstorage = cfd_->current()->storage_info();
975 976

  int num_levels_to_check =
977
      (cfd_->ioptions()->compaction_style != kCompactionStyleFIFO)
978
          ? vstorage->num_levels() - 1
979
          : 1;
S
sdong 已提交
980

981
  // Compaction scores are sorted based on its value. Restore them to the
982 983 984
  // level order
  std::vector<double> compaction_score(number_levels_, 0);
  for (int i = 0; i < num_levels_to_check; ++i) {
985 986
    compaction_score[vstorage->CompactionScoreLevel(i)] =
        vstorage->CompactionScore(i);
987 988 989
  }
  // Count # of files being compacted for each level
  std::vector<int> files_being_compacted(number_levels_, 0);
990
  for (int level = 0; level < number_levels_; ++level) {
991
    for (auto* f : vstorage->LevelFiles(level)) {
992 993 994 995 996 997 998 999 1000
      if (f->being_compacted) {
        ++files_being_compacted[level];
      }
    }
  }

  int total_files = 0;
  int total_files_being_compacted = 0;
  double total_file_size = 0;
1001 1002 1003
  uint64_t flush_ingest = cf_stats_value_[BYTES_FLUSHED];
  uint64_t add_file_ingest = cf_stats_value_[BYTES_INGESTED_ADD_FILE];
  uint64_t curr_ingest = flush_ingest + add_file_ingest;
1004
  for (int level = 0; level < number_levels_; level++) {
S
sdong 已提交
1005
    int files = vstorage->NumLevelFiles(level);
1006 1007 1008
    total_files += files;
    total_files_being_compacted += files_being_compacted[level];
    if (comp_stats_[level].micros > 0 || files > 0) {
1009
      compaction_stats_sum->Add(comp_stats_[level]);
S
sdong 已提交
1010
      total_file_size += vstorage->NumLevelBytes(level);
1011 1012 1013 1014 1015 1016
      uint64_t input_bytes;
      if (level == 0) {
        input_bytes = curr_ingest;
      } else {
        input_bytes = comp_stats_[level].bytes_read_non_output_levels;
      }
1017
      double w_amp =
1018
          (input_bytes == 0)
1019 1020
              ? 0.0
              : static_cast<double>(comp_stats_[level].bytes_written) /
1021
                    input_bytes;
1022 1023 1024 1025 1026
      std::map<LevelStatType, double> level_stats;
      PrepareLevelStats(&level_stats, files, files_being_compacted[level],
                        static_cast<double>(vstorage->NumLevelBytes(level)),
                        compaction_score[level], w_amp, comp_stats_[level]);
      (*levels_stats)[level] = level_stats;
1027 1028
    }
  }
1029 1030 1031 1032 1033 1034 1035 1036 1037 1038
  // Cumulative summary
  double w_amp = compaction_stats_sum->bytes_written /
                 static_cast<double>(curr_ingest + 1);
  // Stats summary across levels
  std::map<LevelStatType, double> sum_stats;
  PrepareLevelStats(&sum_stats, total_files, total_files_being_compacted,
                    total_file_size, 0, w_amp, *compaction_stats_sum);
  (*levels_stats)[-1] = sum_stats;  //  -1 is for the Sum level
}

1039 1040 1041
void InternalStats::DumpCFMapStatsIOStalls(
    std::map<std::string, std::string>* cf_stats) {
  (*cf_stats)["io_stalls.level0_slowdown"] =
1042
      std::to_string(cf_stats_count_[L0_FILE_COUNT_LIMIT_SLOWDOWNS]);
1043
  (*cf_stats)["io_stalls.level0_slowdown_with_compaction"] =
1044
      std::to_string(cf_stats_count_[LOCKED_L0_FILE_COUNT_LIMIT_SLOWDOWNS]);
1045
  (*cf_stats)["io_stalls.level0_numfiles"] =
1046
      std::to_string(cf_stats_count_[L0_FILE_COUNT_LIMIT_STOPS]);
1047
  (*cf_stats)["io_stalls.level0_numfiles_with_compaction"] =
1048
      std::to_string(cf_stats_count_[LOCKED_L0_FILE_COUNT_LIMIT_STOPS]);
1049
  (*cf_stats)["io_stalls.stop_for_pending_compaction_bytes"] =
1050
      std::to_string(cf_stats_count_[PENDING_COMPACTION_BYTES_LIMIT_STOPS]);
1051
  (*cf_stats)["io_stalls.slowdown_for_pending_compaction_bytes"] =
1052
      std::to_string(cf_stats_count_[PENDING_COMPACTION_BYTES_LIMIT_SLOWDOWNS]);
1053
  (*cf_stats)["io_stalls.memtable_compaction"] =
1054
      std::to_string(cf_stats_count_[MEMTABLE_LIMIT_STOPS]);
1055
  (*cf_stats)["io_stalls.memtable_slowdown"] =
1056
      std::to_string(cf_stats_count_[MEMTABLE_LIMIT_SLOWDOWNS]);
1057

1058 1059 1060
  uint64_t total_stop = cf_stats_count_[L0_FILE_COUNT_LIMIT_STOPS] +
                        cf_stats_count_[PENDING_COMPACTION_BYTES_LIMIT_STOPS] +
                        cf_stats_count_[MEMTABLE_LIMIT_STOPS];
1061 1062

  uint64_t total_slowdown =
1063 1064 1065
      cf_stats_count_[L0_FILE_COUNT_LIMIT_SLOWDOWNS] +
      cf_stats_count_[PENDING_COMPACTION_BYTES_LIMIT_SLOWDOWNS] +
      cf_stats_count_[MEMTABLE_LIMIT_SLOWDOWNS];
1066 1067 1068 1069 1070

  (*cf_stats)["io_stalls.total_stop"] = std::to_string(total_stop);
  (*cf_stats)["io_stalls.total_slowdown"] = std::to_string(total_slowdown);
}

1071
void InternalStats::DumpCFStats(std::string* value) {
1072 1073 1074 1075 1076
  DumpCFStatsNoFileHistogram(value);
  DumpCFFileHistogram(value);
}

void InternalStats::DumpCFStatsNoFileHistogram(std::string* value) {
J
Jonathan Lee 已提交
1077
  char buf[2000];
1078 1079 1080 1081 1082 1083 1084
  // Per-ColumnFamily stats
  PrintLevelStatsHeader(buf, sizeof(buf), cfd_->GetName());
  value->append(buf);

  // Print stats for each level
  std::map<int, std::map<LevelStatType, double>> levels_stats;
  CompactionStats compaction_stats_sum(0);
1085 1086
  DumpCFMapStats(&levels_stats, &compaction_stats_sum);
  for (int l = 0; l < number_levels_; ++l) {
1087 1088 1089 1090 1091
    if (levels_stats.find(l) != levels_stats.end()) {
      PrintLevelStats(buf, sizeof(buf), "L" + ToString(l), levels_stats[l]);
      value->append(buf);
    }
  }
1092

1093 1094 1095 1096 1097 1098
  // Print sum of level stats
  PrintLevelStats(buf, sizeof(buf), "Sum", levels_stats[-1]);
  value->append(buf);

  uint64_t flush_ingest = cf_stats_value_[BYTES_FLUSHED];
  uint64_t add_file_ingest = cf_stats_value_[BYTES_INGESTED_ADD_FILE];
1099 1100 1101 1102
  uint64_t ingest_files_addfile = cf_stats_value_[INGESTED_NUM_FILES_TOTAL];
  uint64_t ingest_l0_files_addfile =
      cf_stats_value_[INGESTED_LEVEL0_NUM_FILES_TOTAL];
  uint64_t ingest_keys_addfile = cf_stats_value_[INGESTED_NUM_KEYS_TOTAL];
1103
  // Cumulative summary
1104
  uint64_t total_stall_count =
1105 1106 1107 1108 1109 1110
      cf_stats_count_[L0_FILE_COUNT_LIMIT_SLOWDOWNS] +
      cf_stats_count_[L0_FILE_COUNT_LIMIT_STOPS] +
      cf_stats_count_[PENDING_COMPACTION_BYTES_LIMIT_SLOWDOWNS] +
      cf_stats_count_[PENDING_COMPACTION_BYTES_LIMIT_STOPS] +
      cf_stats_count_[MEMTABLE_LIMIT_STOPS] +
      cf_stats_count_[MEMTABLE_LIMIT_SLOWDOWNS];
1111
  // Interval summary
1112 1113 1114
  uint64_t interval_flush_ingest =
      flush_ingest - cf_stats_snapshot_.ingest_bytes_flush;
  uint64_t interval_add_file_inget =
1115
      add_file_ingest - cf_stats_snapshot_.ingest_bytes_addfile;
1116
  uint64_t interval_ingest =
1117
      interval_flush_ingest + interval_add_file_inget + 1;
1118
  CompactionStats interval_stats(compaction_stats_sum);
1119
  interval_stats.Subtract(cf_stats_snapshot_.comp_stats);
1120 1121
  double w_amp =
      interval_stats.bytes_written / static_cast<double>(interval_ingest);
1122
  PrintLevelStats(buf, sizeof(buf), "Int", 0, 0, 0, 0, w_amp, interval_stats);
1123 1124
  value->append(buf);

1125 1126 1127 1128 1129
  double seconds_up = (env_->NowMicros() - started_at_ + 1) / kMicrosInSec;
  double interval_seconds_up = seconds_up - cf_stats_snapshot_.seconds_up;
  snprintf(buf, sizeof(buf), "Uptime(secs): %.1f total, %.1f interval\n",
           seconds_up, interval_seconds_up);
  value->append(buf);
1130 1131
  snprintf(buf, sizeof(buf), "Flush(GB): cumulative %.3f, interval %.3f\n",
           flush_ingest / kGB, interval_flush_ingest / kGB);
1132
  value->append(buf);
1133 1134
  snprintf(buf, sizeof(buf), "AddFile(GB): cumulative %.3f, interval %.3f\n",
           add_file_ingest / kGB, interval_add_file_inget / kGB);
1135 1136
  value->append(buf);

1137 1138
  uint64_t interval_ingest_files_addfile =
      ingest_files_addfile - cf_stats_snapshot_.ingest_files_addfile;
I
Islam AbdelRahman 已提交
1139 1140
  snprintf(buf, sizeof(buf), "AddFile(Total Files): cumulative %" PRIu64
                             ", interval %" PRIu64 "\n",
1141 1142 1143 1144 1145 1146
           ingest_files_addfile, interval_ingest_files_addfile);
  value->append(buf);

  uint64_t interval_ingest_l0_files_addfile =
      ingest_l0_files_addfile - cf_stats_snapshot_.ingest_l0_files_addfile;
  snprintf(buf, sizeof(buf),
I
Islam AbdelRahman 已提交
1147
           "AddFile(L0 Files): cumulative %" PRIu64 ", interval %" PRIu64 "\n",
1148 1149 1150 1151 1152
           ingest_l0_files_addfile, interval_ingest_l0_files_addfile);
  value->append(buf);

  uint64_t interval_ingest_keys_addfile =
      ingest_keys_addfile - cf_stats_snapshot_.ingest_keys_addfile;
I
Islam AbdelRahman 已提交
1153 1154
  snprintf(buf, sizeof(buf),
           "AddFile(Keys): cumulative %" PRIu64 ", interval %" PRIu64 "\n",
1155 1156 1157
           ingest_keys_addfile, interval_ingest_keys_addfile);
  value->append(buf);

1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198
  // Compact
  uint64_t compact_bytes_read = 0;
  uint64_t compact_bytes_write = 0;
  uint64_t compact_micros = 0;
  for (int level = 0; level < number_levels_; level++) {
    compact_bytes_read += comp_stats_[level].bytes_read_output_level +
                          comp_stats_[level].bytes_read_non_output_levels;
    compact_bytes_write += comp_stats_[level].bytes_written;
    compact_micros += comp_stats_[level].micros;
  }

  snprintf(buf, sizeof(buf),
           "Cumulative compaction: %.2f GB write, %.2f MB/s write, "
           "%.2f GB read, %.2f MB/s read, %.1f seconds\n",
           compact_bytes_write / kGB, compact_bytes_write / kMB / seconds_up,
           compact_bytes_read / kGB, compact_bytes_read / kMB / seconds_up,
           compact_micros / kMicrosInSec);
  value->append(buf);

  // Compaction interval
  uint64_t interval_compact_bytes_write =
      compact_bytes_write - cf_stats_snapshot_.compact_bytes_write;
  uint64_t interval_compact_bytes_read =
      compact_bytes_read - cf_stats_snapshot_.compact_bytes_read;
  uint64_t interval_compact_micros =
      compact_micros - cf_stats_snapshot_.compact_micros;

  snprintf(
      buf, sizeof(buf),
      "Interval compaction: %.2f GB write, %.2f MB/s write, "
      "%.2f GB read, %.2f MB/s read, %.1f seconds\n",
      interval_compact_bytes_write / kGB,
      interval_compact_bytes_write / kMB / std::max(interval_seconds_up, 0.001),
      interval_compact_bytes_read / kGB,
      interval_compact_bytes_read / kMB / std::max(interval_seconds_up, 0.001),
      interval_compact_micros / kMicrosInSec);
  value->append(buf);
  cf_stats_snapshot_.compact_bytes_write = compact_bytes_write;
  cf_stats_snapshot_.compact_bytes_read = compact_bytes_read;
  cf_stats_snapshot_.compact_micros = compact_micros;

1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224
  snprintf(buf, sizeof(buf),
           "Stalls(count): %" PRIu64
           " level0_slowdown, "
           "%" PRIu64
           " level0_slowdown_with_compaction, "
           "%" PRIu64
           " level0_numfiles, "
           "%" PRIu64
           " level0_numfiles_with_compaction, "
           "%" PRIu64
           " stop for pending_compaction_bytes, "
           "%" PRIu64
           " slowdown for pending_compaction_bytes, "
           "%" PRIu64
           " memtable_compaction, "
           "%" PRIu64
           " memtable_slowdown, "
           "interval %" PRIu64 " total count\n",
           cf_stats_count_[L0_FILE_COUNT_LIMIT_SLOWDOWNS],
           cf_stats_count_[LOCKED_L0_FILE_COUNT_LIMIT_SLOWDOWNS],
           cf_stats_count_[L0_FILE_COUNT_LIMIT_STOPS],
           cf_stats_count_[LOCKED_L0_FILE_COUNT_LIMIT_STOPS],
           cf_stats_count_[PENDING_COMPACTION_BYTES_LIMIT_STOPS],
           cf_stats_count_[PENDING_COMPACTION_BYTES_LIMIT_SLOWDOWNS],
           cf_stats_count_[MEMTABLE_LIMIT_STOPS],
           cf_stats_count_[MEMTABLE_LIMIT_SLOWDOWNS],
1225
           total_stall_count - cf_stats_snapshot_.stall_count);
1226 1227
  value->append(buf);

1228
  cf_stats_snapshot_.seconds_up = seconds_up;
1229
  cf_stats_snapshot_.ingest_bytes_flush = flush_ingest;
1230 1231 1232 1233
  cf_stats_snapshot_.ingest_bytes_addfile = add_file_ingest;
  cf_stats_snapshot_.ingest_files_addfile = ingest_files_addfile;
  cf_stats_snapshot_.ingest_l0_files_addfile = ingest_l0_files_addfile;
  cf_stats_snapshot_.ingest_keys_addfile = ingest_keys_addfile;
1234
  cf_stats_snapshot_.comp_stats = compaction_stats_sum;
1235 1236 1237
  cf_stats_snapshot_.stall_count = total_stall_count;
}

1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254
void InternalStats::DumpCFFileHistogram(std::string* value) {
  char buf[2000];
  snprintf(buf, sizeof(buf),
           "\n** File Read Latency Histogram By Level [%s] **\n",
           cfd_->GetName().c_str());
  value->append(buf);

  for (int level = 0; level < number_levels_; level++) {
    if (!file_read_latency_[level].Empty()) {
      char buf2[5000];
      snprintf(buf2, sizeof(buf2),
               "** Level %d read latency histogram (micros):\n%s\n", level,
               file_read_latency_[level].ToString().c_str());
      value->append(buf2);
    }
  }
}
1255 1256 1257

#else

1258
const DBPropertyInfo* GetPropertyInfo(const Slice& property) { return nullptr; }
1259 1260 1261

#endif  // !ROCKSDB_LITE

I
Igor Canadi 已提交
1262
}  // namespace rocksdb