Blob DB: blob_dump to show uncompressed values

Summary:
Make blob_dump tool able to show uncompressed values if the blob file is compressed. Also show total compressed vs. raw size at the end if --show_summary is provided.
Closes https://github.com/facebook/rocksdb/pull/3633

Differential Revision: D7348926

Pulled By: yiwu-arbug

fbshipit-source-id: ca709cb4ed5cf6a550ff2987df8033df81516f8e

tools/blob_dump.cc

@@ -27,9 +27,13 @@ int main(int argc, char** argv) {
       {"file", required_argument, nullptr, 'f'},
       {"show_key", optional_argument, nullptr, 'k'},
       {"show_blob", optional_argument, nullptr, 'b'},
+      {"show_uncompressed_blob", optional_argument, nullptr, 'r'},
+      {"show_summary", optional_argument, nullptr, 's'},
   };
   DisplayType show_key = DisplayType::kRaw;
   DisplayType show_blob = DisplayType::kNone;
+  DisplayType show_uncompressed_blob = DisplayType::kNone;
+  bool show_summary = false;
   std::string file;
   while (true) {
     int c = getopt_long(argc, argv, "hk::b::f:", options, nullptr);
@@ -42,7 +46,9 @@ int main(int argc, char** argv) {
         fprintf(stdout,
                 "Usage: blob_dump --file=filename "
                 "[--show_key[=none|raw|hex|detail]] "
-                "[--show_blob[=none|raw|hex|detail]]\n");
+                "[--show_blob[=none|raw|hex|detail]] "
+                "[--show_uncompressed_blob[=none|raw|hex|detail]] "
+                "[--show_summary]\n");
         return 0;
       case 'f':
         file = optarg;
@@ -64,16 +70,31 @@ int main(int argc, char** argv) {
           }
           show_blob = display_types.at(arg_str);
         } else {
-          show_blob = DisplayType::kDetail;
+          show_blob = DisplayType::kHex;
         }
         break;
+      case 'r':
+        if (optarg) {
+          if (display_types.count(arg_str) == 0) {
+            fprintf(stderr, "Unrecognized blob display type.\n");
+            return -1;
+          }
+          show_uncompressed_blob = display_types.at(arg_str);
+        } else {
+          show_uncompressed_blob = DisplayType::kHex;
+        }
+        break;
+      case 's':
+        show_summary = true;
+        break;
       default:
         fprintf(stderr, "Unrecognized option.\n");
         return -1;
     }
   }
   BlobDumpTool tool;
-  Status s = tool.Run(file, show_key, show_blob);
+  Status s =
+      tool.Run(file, show_key, show_blob, show_uncompressed_blob, show_summary);
   if (!s.ok()) {
     fprintf(stderr, "Failed: %s\n", s.ToString().c_str());
     return -1;

utilities/blob_db/blob_dump_tool.cc

@@ -17,6 +17,7 @@
 #include "port/port.h"
 #include "rocksdb/convenience.h"
 #include "rocksdb/env.h"
+#include "table/format.h"
 #include "util/coding.h"
 #include "util/string_util.h"
 
@@ -27,7 +28,9 @@ BlobDumpTool::BlobDumpTool()
     : reader_(nullptr), buffer_(nullptr), buffer_size_(0) {}
 
 Status BlobDumpTool::Run(const std::string& filename, DisplayType show_key,
-                         DisplayType show_blob) {
+                         DisplayType show_blob,
+                         DisplayType show_uncompressed_blob,
+                         bool show_summary) {
   Status s;
   Env* env = Env::Default();
   s = env->FileExists(filename);
@@ -50,7 +53,8 @@ Status BlobDumpTool::Run(const std::string& filename, DisplayType show_key,
   reader_.reset(new RandomAccessFileReader(std::move(file), filename));
   uint64_t offset = 0;
   uint64_t footer_offset = 0;
-  s = DumpBlobLogHeader(&offset);
+  CompressionType compression = kNoCompression;
+  s = DumpBlobLogHeader(&offset, &compression);
   if (!s.ok()) {
     return s;
   }
@@ -58,14 +62,30 @@ Status BlobDumpTool::Run(const std::string& filename, DisplayType show_key,
   if (!s.ok()) {
     return s;
   }
-  if (show_key != DisplayType::kNone) {
+  uint64_t total_records = 0;
+  uint64_t total_key_size = 0;
+  uint64_t total_blob_size = 0;
+  uint64_t total_uncompressed_blob_size = 0;
+  if (show_key != DisplayType::kNone || show_summary) {
     while (offset < footer_offset) {
-      s = DumpRecord(show_key, show_blob, &offset);
+      s = DumpRecord(show_key, show_blob, show_uncompressed_blob, show_summary,
+                     compression, &offset, &total_records, &total_key_size,
+                     &total_blob_size, &total_uncompressed_blob_size);
       if (!s.ok()) {
-        return s;
+        break;
       }
     }
   }
+  if (show_summary) {
+    fprintf(stdout, "Summary:\n");
+    fprintf(stdout, "  total records: %" PRIu64 "\n", total_records);
+    fprintf(stdout, "  total key size: %" PRIu64 "\n", total_key_size);
+    fprintf(stdout, "  total blob size: %" PRIu64 "\n", total_blob_size);
+    if (compression != kNoCompression) {
+      fprintf(stdout, "  total raw blob size: %" PRIu64 "\n",
+              total_uncompressed_blob_size);
+    }
+  }
   return s;
 }
 
@@ -89,7 +109,8 @@ Status BlobDumpTool::Read(uint64_t offset, size_t size, Slice* result) {
   return s;
 }
 
-Status BlobDumpTool::DumpBlobLogHeader(uint64_t* offset) {
+Status BlobDumpTool::DumpBlobLogHeader(uint64_t* offset,
+                                       CompressionType* compression) {
   Slice slice;
   Status s = Read(0, BlobLogHeader::kSize, &slice);
   if (!s.ok()) {
@@ -114,6 +135,7 @@ Status BlobDumpTool::DumpBlobLogHeader(uint64_t* offset) {
   fprintf(stdout, "  Expiration range : %s\n",
           GetString(header.expiration_range).c_str());
   *offset = BlobLogHeader::kSize;
+  *compression = header.compression;
   return s;
 }
 
@@ -136,7 +158,7 @@ Status BlobDumpTool::DumpBlobLogFooter(uint64_t file_size,
   BlobLogFooter footer;
   s = footer.DecodeFrom(slice);
   if (!s.ok()) {
-    return s;
+    return no_footer();
   }
   fprintf(stdout, "Blob log footer:\n");
   fprintf(stdout, "  Blob count       : %" PRIu64 "\n", footer.blob_count);
@@ -146,9 +168,16 @@ Status BlobDumpTool::DumpBlobLogFooter(uint64_t file_size,
 }
 
 Status BlobDumpTool::DumpRecord(DisplayType show_key, DisplayType show_blob,
-                                uint64_t* offset) {
-  fprintf(stdout, "Read record with offset 0x%" PRIx64 " (%" PRIu64 "):\n",
-          *offset, *offset);
+                                DisplayType show_uncompressed_blob,
+                                bool show_summary, CompressionType compression,
+                                uint64_t* offset, uint64_t* total_records,
+                                uint64_t* total_key_size,
+                                uint64_t* total_blob_size,
+                                uint64_t* total_uncompressed_blob_size) {
+  if (show_key != DisplayType::kNone) {
+    fprintf(stdout, "Read record with offset 0x%" PRIx64 " (%" PRIu64 "):\n",
+            *offset, *offset);
+  }
   Slice slice;
   Status s = Read(*offset, BlobLogRecord::kHeaderSize, &slice);
   if (!s.ok()) {
@@ -161,14 +190,30 @@ Status BlobDumpTool::DumpRecord(DisplayType show_key, DisplayType show_blob,
   }
   uint64_t key_size = record.key_size;
   uint64_t value_size = record.value_size;
-  fprintf(stdout, "  key size   : %" PRIu64 "\n", key_size);
-  fprintf(stdout, "  value size : %" PRIu64 "\n", value_size);
-  fprintf(stdout, "  expiration : %" PRIu64 "\n", record.expiration);
+  if (show_key != DisplayType::kNone) {
+    fprintf(stdout, "  key size   : %" PRIu64 "\n", key_size);
+    fprintf(stdout, "  value size : %" PRIu64 "\n", value_size);
+    fprintf(stdout, "  expiration : %" PRIu64 "\n", record.expiration);
+  }
   *offset += BlobLogRecord::kHeaderSize;
   s = Read(*offset, key_size + value_size, &slice);
   if (!s.ok()) {
     return s;
   }
+  // Decompress value
+  std::string uncompressed_value;
+  if (compression != kNoCompression &&
+      (show_uncompressed_blob != DisplayType::kNone || show_summary)) {
+    BlockContents contents;
+    s = UncompressBlockContentsForCompressionType(
+        slice.data() + key_size, value_size, &contents,
+        2 /*compress_format_version*/, Slice(), compression,
+        ImmutableCFOptions(Options()));
+    if (!s.ok()) {
+      return s;
+    }
+    uncompressed_value = contents.data.ToString();
+  }
   if (show_key != DisplayType::kNone) {
     fprintf(stdout, "  key        : ");
     DumpSlice(Slice(slice.data(), key_size), show_key);
@@ -176,8 +221,16 @@ Status BlobDumpTool::DumpRecord(DisplayType show_key, DisplayType show_blob,
       fprintf(stdout, "  blob       : ");
       DumpSlice(Slice(slice.data() + key_size, value_size), show_blob);
     }
+    if (show_uncompressed_blob != DisplayType::kNone) {
+      fprintf(stdout, "  raw blob   : ");
+      DumpSlice(Slice(uncompressed_value), show_uncompressed_blob);
+    }
   }
   *offset += key_size + value_size;
+  *total_records += 1;
+  *total_key_size += key_size;
+  *total_blob_size += value_size;
+  *total_uncompressed_blob_size += uncompressed_value.size();
   return s;
 }
 

utilities/blob_db/blob_dump_tool.h

@@ -27,8 +27,9 @@ class BlobDumpTool {
 
   BlobDumpTool();
 
-  Status Run(const std::string& filename, DisplayType key_type,
-             DisplayType blob_type);
+  Status Run(const std::string& filename, DisplayType show_key,
+             DisplayType show_blob, DisplayType show_uncompressed_blob,
+             bool show_summary);
 
  private:
   std::unique_ptr<RandomAccessFileReader> reader_;
@@ -36,10 +37,14 @@ class BlobDumpTool {
   size_t buffer_size_;
 
   Status Read(uint64_t offset, size_t size, Slice* result);
-  Status DumpBlobLogHeader(uint64_t* offset);
+  Status DumpBlobLogHeader(uint64_t* offset, CompressionType* compression);
   Status DumpBlobLogFooter(uint64_t file_size, uint64_t* footer_offset);
   Status DumpRecord(DisplayType show_key, DisplayType show_blob,
-                    uint64_t* offset);
+                    DisplayType show_uncompressed_blob, bool show_summary,
+                    CompressionType compression, uint64_t* offset,
+                    uint64_t* total_records, uint64_t* total_key_size,
+                    uint64_t* total_blob_size,
+                    uint64_t* total_uncompressed_blob_size);
   void DumpSlice(const Slice s, DisplayType type);
 
   template <class T>