From 36a9f229313381b723288a87d684a099212783c5 Mon Sep 17 00:00:00 2001 From: Yi Wu Date: Thu, 5 Apr 2018 11:00:52 -0700 Subject: [PATCH] Blob DB: blob_dump to show uncompressed values Summary: Make blob_dump tool able to show uncompressed values if the blob file is compressed. Also show total compressed vs. raw size at the end if --show_summary is provided. Closes https://github.com/facebook/rocksdb/pull/3633 Differential Revision: D7348926 Pulled By: yiwu-arbug fbshipit-source-id: ca709cb4ed5cf6a550ff2987df8033df81516f8e --- tools/blob_dump.cc | 27 ++++++++-- utilities/blob_db/blob_dump_tool.cc | 79 ++++++++++++++++++++++++----- utilities/blob_db/blob_dump_tool.h | 13 +++-- 3 files changed, 99 insertions(+), 20 deletions(-) diff --git a/tools/blob_dump.cc b/tools/blob_dump.cc index 73601f2d8..58964e4f8 100644 --- a/tools/blob_dump.cc +++ b/tools/blob_dump.cc @@ -27,9 +27,13 @@ int main(int argc, char** argv) { {"file", required_argument, nullptr, 'f'}, {"show_key", optional_argument, nullptr, 'k'}, {"show_blob", optional_argument, nullptr, 'b'}, + {"show_uncompressed_blob", optional_argument, nullptr, 'r'}, + {"show_summary", optional_argument, nullptr, 's'}, }; DisplayType show_key = DisplayType::kRaw; DisplayType show_blob = DisplayType::kNone; + DisplayType show_uncompressed_blob = DisplayType::kNone; + bool show_summary = false; std::string file; while (true) { int c = getopt_long(argc, argv, "hk::b::f:", options, nullptr); @@ -42,7 +46,9 @@ int main(int argc, char** argv) { fprintf(stdout, "Usage: blob_dump --file=filename " "[--show_key[=none|raw|hex|detail]] " - "[--show_blob[=none|raw|hex|detail]]\n"); + "[--show_blob[=none|raw|hex|detail]] " + "[--show_uncompressed_blob[=none|raw|hex|detail]] " + "[--show_summary]\n"); return 0; case 'f': file = optarg; @@ -64,16 +70,31 @@ int main(int argc, char** argv) { } show_blob = display_types.at(arg_str); } else { - show_blob = DisplayType::kDetail; + show_blob = DisplayType::kHex; } break; + case 'r': + if (optarg) { + if (display_types.count(arg_str) == 0) { + fprintf(stderr, "Unrecognized blob display type.\n"); + return -1; + } + show_uncompressed_blob = display_types.at(arg_str); + } else { + show_uncompressed_blob = DisplayType::kHex; + } + break; + case 's': + show_summary = true; + break; default: fprintf(stderr, "Unrecognized option.\n"); return -1; } } BlobDumpTool tool; - Status s = tool.Run(file, show_key, show_blob); + Status s = + tool.Run(file, show_key, show_blob, show_uncompressed_blob, show_summary); if (!s.ok()) { fprintf(stderr, "Failed: %s\n", s.ToString().c_str()); return -1; diff --git a/utilities/blob_db/blob_dump_tool.cc b/utilities/blob_db/blob_dump_tool.cc index 7d3895bcc..c527bc9d5 100644 --- a/utilities/blob_db/blob_dump_tool.cc +++ b/utilities/blob_db/blob_dump_tool.cc @@ -17,6 +17,7 @@ #include "port/port.h" #include "rocksdb/convenience.h" #include "rocksdb/env.h" +#include "table/format.h" #include "util/coding.h" #include "util/string_util.h" @@ -27,7 +28,9 @@ BlobDumpTool::BlobDumpTool() : reader_(nullptr), buffer_(nullptr), buffer_size_(0) {} Status BlobDumpTool::Run(const std::string& filename, DisplayType show_key, - DisplayType show_blob) { + DisplayType show_blob, + DisplayType show_uncompressed_blob, + bool show_summary) { Status s; Env* env = Env::Default(); s = env->FileExists(filename); @@ -50,7 +53,8 @@ Status BlobDumpTool::Run(const std::string& filename, DisplayType show_key, reader_.reset(new RandomAccessFileReader(std::move(file), filename)); uint64_t offset = 0; uint64_t footer_offset = 0; - s = DumpBlobLogHeader(&offset); + CompressionType compression = kNoCompression; + s = DumpBlobLogHeader(&offset, &compression); if (!s.ok()) { return s; } @@ -58,14 +62,30 @@ Status BlobDumpTool::Run(const std::string& filename, DisplayType show_key, if (!s.ok()) { return s; } - if (show_key != DisplayType::kNone) { + uint64_t total_records = 0; + uint64_t total_key_size = 0; + uint64_t total_blob_size = 0; + uint64_t total_uncompressed_blob_size = 0; + if (show_key != DisplayType::kNone || show_summary) { while (offset < footer_offset) { - s = DumpRecord(show_key, show_blob, &offset); + s = DumpRecord(show_key, show_blob, show_uncompressed_blob, show_summary, + compression, &offset, &total_records, &total_key_size, + &total_blob_size, &total_uncompressed_blob_size); if (!s.ok()) { - return s; + break; } } } + if (show_summary) { + fprintf(stdout, "Summary:\n"); + fprintf(stdout, " total records: %" PRIu64 "\n", total_records); + fprintf(stdout, " total key size: %" PRIu64 "\n", total_key_size); + fprintf(stdout, " total blob size: %" PRIu64 "\n", total_blob_size); + if (compression != kNoCompression) { + fprintf(stdout, " total raw blob size: %" PRIu64 "\n", + total_uncompressed_blob_size); + } + } return s; } @@ -89,7 +109,8 @@ Status BlobDumpTool::Read(uint64_t offset, size_t size, Slice* result) { return s; } -Status BlobDumpTool::DumpBlobLogHeader(uint64_t* offset) { +Status BlobDumpTool::DumpBlobLogHeader(uint64_t* offset, + CompressionType* compression) { Slice slice; Status s = Read(0, BlobLogHeader::kSize, &slice); if (!s.ok()) { @@ -114,6 +135,7 @@ Status BlobDumpTool::DumpBlobLogHeader(uint64_t* offset) { fprintf(stdout, " Expiration range : %s\n", GetString(header.expiration_range).c_str()); *offset = BlobLogHeader::kSize; + *compression = header.compression; return s; } @@ -136,7 +158,7 @@ Status BlobDumpTool::DumpBlobLogFooter(uint64_t file_size, BlobLogFooter footer; s = footer.DecodeFrom(slice); if (!s.ok()) { - return s; + return no_footer(); } fprintf(stdout, "Blob log footer:\n"); fprintf(stdout, " Blob count : %" PRIu64 "\n", footer.blob_count); @@ -146,9 +168,16 @@ Status BlobDumpTool::DumpBlobLogFooter(uint64_t file_size, } Status BlobDumpTool::DumpRecord(DisplayType show_key, DisplayType show_blob, - uint64_t* offset) { - fprintf(stdout, "Read record with offset 0x%" PRIx64 " (%" PRIu64 "):\n", - *offset, *offset); + DisplayType show_uncompressed_blob, + bool show_summary, CompressionType compression, + uint64_t* offset, uint64_t* total_records, + uint64_t* total_key_size, + uint64_t* total_blob_size, + uint64_t* total_uncompressed_blob_size) { + if (show_key != DisplayType::kNone) { + fprintf(stdout, "Read record with offset 0x%" PRIx64 " (%" PRIu64 "):\n", + *offset, *offset); + } Slice slice; Status s = Read(*offset, BlobLogRecord::kHeaderSize, &slice); if (!s.ok()) { @@ -161,14 +190,30 @@ Status BlobDumpTool::DumpRecord(DisplayType show_key, DisplayType show_blob, } uint64_t key_size = record.key_size; uint64_t value_size = record.value_size; - fprintf(stdout, " key size : %" PRIu64 "\n", key_size); - fprintf(stdout, " value size : %" PRIu64 "\n", value_size); - fprintf(stdout, " expiration : %" PRIu64 "\n", record.expiration); + if (show_key != DisplayType::kNone) { + fprintf(stdout, " key size : %" PRIu64 "\n", key_size); + fprintf(stdout, " value size : %" PRIu64 "\n", value_size); + fprintf(stdout, " expiration : %" PRIu64 "\n", record.expiration); + } *offset += BlobLogRecord::kHeaderSize; s = Read(*offset, key_size + value_size, &slice); if (!s.ok()) { return s; } + // Decompress value + std::string uncompressed_value; + if (compression != kNoCompression && + (show_uncompressed_blob != DisplayType::kNone || show_summary)) { + BlockContents contents; + s = UncompressBlockContentsForCompressionType( + slice.data() + key_size, value_size, &contents, + 2 /*compress_format_version*/, Slice(), compression, + ImmutableCFOptions(Options())); + if (!s.ok()) { + return s; + } + uncompressed_value = contents.data.ToString(); + } if (show_key != DisplayType::kNone) { fprintf(stdout, " key : "); DumpSlice(Slice(slice.data(), key_size), show_key); @@ -176,8 +221,16 @@ Status BlobDumpTool::DumpRecord(DisplayType show_key, DisplayType show_blob, fprintf(stdout, " blob : "); DumpSlice(Slice(slice.data() + key_size, value_size), show_blob); } + if (show_uncompressed_blob != DisplayType::kNone) { + fprintf(stdout, " raw blob : "); + DumpSlice(Slice(uncompressed_value), show_uncompressed_blob); + } } *offset += key_size + value_size; + *total_records += 1; + *total_key_size += key_size; + *total_blob_size += value_size; + *total_uncompressed_blob_size += uncompressed_value.size(); return s; } diff --git a/utilities/blob_db/blob_dump_tool.h b/utilities/blob_db/blob_dump_tool.h index abba91dca..e91feffa7 100644 --- a/utilities/blob_db/blob_dump_tool.h +++ b/utilities/blob_db/blob_dump_tool.h @@ -27,8 +27,9 @@ class BlobDumpTool { BlobDumpTool(); - Status Run(const std::string& filename, DisplayType key_type, - DisplayType blob_type); + Status Run(const std::string& filename, DisplayType show_key, + DisplayType show_blob, DisplayType show_uncompressed_blob, + bool show_summary); private: std::unique_ptr reader_; @@ -36,10 +37,14 @@ class BlobDumpTool { size_t buffer_size_; Status Read(uint64_t offset, size_t size, Slice* result); - Status DumpBlobLogHeader(uint64_t* offset); + Status DumpBlobLogHeader(uint64_t* offset, CompressionType* compression); Status DumpBlobLogFooter(uint64_t file_size, uint64_t* footer_offset); Status DumpRecord(DisplayType show_key, DisplayType show_blob, - uint64_t* offset); + DisplayType show_uncompressed_blob, bool show_summary, + CompressionType compression, uint64_t* offset, + uint64_t* total_records, uint64_t* total_key_size, + uint64_t* total_blob_size, + uint64_t* total_uncompressed_blob_size); void DumpSlice(const Slice s, DisplayType type); template -- GitLab