From 0831a35994c3b31ecdc1bb9462b1cde1d5636bd7 Mon Sep 17 00:00:00 2001 From: sdong Date: Wed, 18 Mar 2015 16:11:02 -0700 Subject: [PATCH] Add a DB Property For Number of Deletions in Memtables Summary: Add a DB property for number of deletions in memtables. It can sometimes help people debug slowness because of too many deletes. Test Plan: Add test cases. Reviewers: rven, yhchiang, kradhakrishnan, igor Reviewed By: igor Subscribers: leveldb, dhruba, yoshinorim Differential Revision: https://reviews.facebook.net/D35247 --- db/db_test.cc | 33 +++++++++++++++++++++++++++++++-- db/internal_stats.cc | 21 ++++++++++++++++++--- db/internal_stats.h | 6 +++++- db/memtable.cc | 9 +++++++-- db/memtable.h | 5 ++++- db/memtable_list.cc | 10 +++++++++- db/memtable_list.h | 2 ++ include/rocksdb/db.h | 4 ++++ 8 files changed, 80 insertions(+), 10 deletions(-) diff --git a/db/db_test.cc b/db/db_test.cc index b56030f0b..d0b84c72b 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -3059,6 +3059,35 @@ TEST_F(DBTest, NumImmutableMemTable) { // "200" is the size of the metadata of an empty skiplist, this would // break if we change the default skiplist implementation ASSERT_EQ(num, "200"); + + uint64_t int_num; + uint64_t base_total_size; + ASSERT_TRUE(dbfull()->GetIntProperty( + handles_[1], "rocksdb.estimate-num-keys", &base_total_size)); + + ASSERT_OK(dbfull()->Delete(writeOpt, handles_[1], "k2")); + ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "k3", "")); + ASSERT_OK(dbfull()->Delete(writeOpt, handles_[1], "k3")); + ASSERT_TRUE(dbfull()->GetIntProperty( + handles_[1], "rocksdb.num-deletes-active-mem-table", &int_num)); + ASSERT_EQ(int_num, 2U); + ASSERT_TRUE(dbfull()->GetIntProperty( + handles_[1], "rocksdb.num-entries-active-mem-table", &int_num)); + ASSERT_EQ(int_num, 3U); + + ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "k2", big_value)); + ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "k2", big_value)); + ASSERT_TRUE(dbfull()->GetIntProperty( + handles_[1], "rocksdb.num-entries-imm-mem-tables", &int_num)); + ASSERT_EQ(int_num, 4U); + ASSERT_TRUE(dbfull()->GetIntProperty( + handles_[1], "rocksdb.num-deletes-imm-mem-tables", &int_num)); + ASSERT_EQ(int_num, 2U); + + ASSERT_TRUE(dbfull()->GetIntProperty( + handles_[1], "rocksdb.estimate-num-keys", &int_num)); + ASSERT_EQ(int_num, base_total_size + 1); + SetPerfLevel(kDisable); ASSERT_TRUE(GetPerfLevel() == kDisable); } while (ChangeCompactOptions()); @@ -3198,7 +3227,7 @@ TEST_F(DBTest, GetProperty) { ASSERT_TRUE(dbfull()->GetProperty("rocksdb.compaction-pending", &num)); ASSERT_EQ(num, "0"); ASSERT_TRUE(dbfull()->GetProperty("rocksdb.estimate-num-keys", &num)); - ASSERT_EQ(num, "4"); + ASSERT_EQ(num, "2"); // Verify the same set of properties through GetIntProperty ASSERT_TRUE( dbfull()->GetIntProperty("rocksdb.num-immutable-mem-table", &int_num)); @@ -3209,7 +3238,7 @@ TEST_F(DBTest, GetProperty) { ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.compaction-pending", &int_num)); ASSERT_EQ(int_num, 0U); ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.estimate-num-keys", &int_num)); - ASSERT_EQ(int_num, 4U); + ASSERT_EQ(int_num, 2U); ASSERT_TRUE( dbfull()->GetIntProperty("rocksdb.estimate-table-readers-mem", &int_num)); diff --git a/db/internal_stats.cc b/db/internal_stats.cc index 33866711c..a89554449 100644 --- a/db/internal_stats.cc +++ b/db/internal_stats.cc @@ -126,6 +126,10 @@ DBPropertyType GetPropertyType(const Slice& property, bool* is_int_property, return kNumEntriesInMutableMemtable; } else if (in == "num-entries-imm-mem-tables") { return kNumEntriesInImmutableMemtable; + } else if (in == "num-deletes-active-mem-table") { + return kNumDeletesInMutableMemtable; + } else if (in == "num-deletes-imm-mem-tables") { + return kNumDeletesInImmutableMemtable; } else if (in == "estimate-num-keys") { return kEstimatedNumKeys; } else if (in == "estimate-table-readers-mem") { @@ -256,17 +260,28 @@ bool InternalStats::GetIntProperty(DBPropertyType property_type, return true; case kNumEntriesInMutableMemtable: // Current number of entires in the active memtable - *value = cfd_->mem()->GetNumEntries(); + *value = cfd_->mem()->num_entries(); return true; case kNumEntriesInImmutableMemtable: // Current number of entries in the immutable memtables *value = cfd_->imm()->current()->GetTotalNumEntries(); return true; + case kNumDeletesInMutableMemtable: + // Current number of entires in the active memtable + *value = cfd_->mem()->num_deletes(); + return true; + case kNumDeletesInImmutableMemtable: + // Current number of entries in the immutable memtables + *value = cfd_->imm()->current()->GetTotalNumDeletes(); + return true; case kEstimatedNumKeys: // Estimate number of entries in the column family: // Use estimated entries in tables + total entries in memtables. - *value = cfd_->mem()->GetNumEntries() + - cfd_->imm()->current()->GetTotalNumEntries() + + *value = cfd_->mem()->num_entries() + + cfd_->imm()->current()->GetTotalNumEntries() - + (cfd_->mem()->num_deletes() + + cfd_->imm()->current()->GetTotalNumDeletes()) * + 2 + vstorage->GetEstimatedActiveKeys(); return true; case kNumSnapshots: diff --git a/db/internal_stats.h b/db/internal_stats.h index e72fc75bc..ac235f641 100644 --- a/db/internal_stats.h +++ b/db/internal_stats.h @@ -40,10 +40,14 @@ enum DBPropertyType : uint32_t { kCurSizeActiveMemTable, // Return current size of the active memtable kCurSizeAllMemTables, // Return current size of all (active + immutable) // memtables - kNumEntriesInMutableMemtable, // Return number of entries in the mutable + kNumEntriesInMutableMemtable, // Return number of deletes in the mutable // memtable. kNumEntriesInImmutableMemtable, // Return sum of number of entries in all // the immutable mem tables. + kNumDeletesInMutableMemtable, // Return number of entries in the mutable + // memtable. + kNumDeletesInImmutableMemtable, // Return sum of number of deletes in all + // the immutable mem tables. kEstimatedNumKeys, // Estimated total number of keys in the database. kEstimatedUsageByTableReaders, // Estimated memory by table readers. kIsFileDeletionEnabled, // Equals disable_delete_obsolete_files_, diff --git a/db/memtable.cc b/db/memtable.cc index 4678e6adf..f94f97ce8 100644 --- a/db/memtable.cc +++ b/db/memtable.cc @@ -65,13 +65,15 @@ MemTable::MemTable(const InternalKeyComparator& cmp, comparator_, &allocator_, ioptions.prefix_extractor, ioptions.info_log)), num_entries_(0), + num_deletes_(0), flush_in_progress_(false), flush_completed_(false), file_number_(0), first_seqno_(0), mem_next_logfile_number_(0), - locks_(moptions_.inplace_update_support ? - moptions_.inplace_update_num_locks : 0), + locks_(moptions_.inplace_update_support + ? moptions_.inplace_update_num_locks + : 0), prefix_extractor_(ioptions.prefix_extractor), should_flush_(ShouldFlushNow()), flush_scheduled_(false) { @@ -313,6 +315,9 @@ void MemTable::Add(SequenceNumber s, ValueType type, assert((unsigned)(p + val_size - buf) == (unsigned)encoded_len); table_->Insert(handle); num_entries_++; + if (type == kTypeDeletion) { + num_deletes_++; + } if (prefix_bloom_) { assert(prefix_extractor_); diff --git a/db/memtable.h b/db/memtable.h index 0b979fc41..4234d5dfd 100644 --- a/db/memtable.h +++ b/db/memtable.h @@ -164,7 +164,9 @@ class MemTable { size_t CountSuccessiveMergeEntries(const LookupKey& key); // Get total number of entries in the mem table. - uint64_t GetNumEntries() const { return num_entries_; } + uint64_t num_entries() const { return num_entries_; } + + uint64_t num_deletes() const { return num_deletes_; } // Returns the edits area that is needed for flushing the memtable VersionEdit* GetEdits() { return &edit_; } @@ -227,6 +229,7 @@ class MemTable { unique_ptr table_; uint64_t num_entries_; + uint64_t num_deletes_; // These are used to manage memtable flushes to storage bool flush_in_progress_; // started the flush diff --git a/db/memtable_list.cc b/db/memtable_list.cc index 73c1e3d59..8adbc65e6 100644 --- a/db/memtable_list.cc +++ b/db/memtable_list.cc @@ -96,7 +96,15 @@ void MemTableListVersion::AddIterators( uint64_t MemTableListVersion::GetTotalNumEntries() const { uint64_t total_num = 0; for (auto& m : memlist_) { - total_num += m->GetNumEntries(); + total_num += m->num_entries(); + } + return total_num; +} + +uint64_t MemTableListVersion::GetTotalNumDeletes() const { + uint64_t total_num = 0; + for (auto& m : memlist_) { + total_num += m->num_deletes(); } return total_num; } diff --git a/db/memtable_list.h b/db/memtable_list.h index 30382eac6..6573728a1 100644 --- a/db/memtable_list.h +++ b/db/memtable_list.h @@ -57,6 +57,8 @@ class MemTableListVersion { uint64_t GetTotalNumEntries() const; + uint64_t GetTotalNumDeletes() const; + private: // REQUIRE: m is mutable memtable void Add(MemTable* m); diff --git a/include/rocksdb/db.h b/include/rocksdb/db.h index ab165a53f..77a7cf57e 100644 --- a/include/rocksdb/db.h +++ b/include/rocksdb/db.h @@ -319,6 +319,8 @@ class DB { // "rocksdb.cur-size-all-mem-tables" // "rocksdb.num-entries-active-mem-table" // "rocksdb.num-entries-imm-mem-tables" + // "rocksdb.num-deletes-active-mem-table" + // "rocksdb.num-deletes-imm-mem-tables" // "rocksdb.estimate-num-keys" - estimated keys in the column family // "rocksdb.estimate-table-readers-mem" - estimated memory used for reding // SST tables, that is not counted as a part of block cache. @@ -346,6 +348,8 @@ class DB { // "rocksdb.cur-size-all-mem-tables" // "rocksdb.num-entries-active-mem-table" // "rocksdb.num-entries-imm-mem-tables" + // "rocksdb.num-deletes-active-mem-table" + // "rocksdb.num-deletes-imm-mem-tables" // "rocksdb.estimate-num-keys" // "rocksdb.estimate-table-readers-mem" // "rocksdb.is-file-deletions-enabled" -- GitLab