diff --git a/db/column_family.h b/db/column_family.h index 5fe593a84b6ffccb8fe96596d8549465a7969c10..05f126ae6baf57e5c612d2ee747a46feebe225a3 100644 --- a/db/column_family.h +++ b/db/column_family.h @@ -705,12 +705,12 @@ class ColumnFamilySet { Version* dummy_version, const ColumnFamilyOptions& options); - const std::unordered_map& - GetRunningColumnFamiliesTimestampSize() const { + const UnorderedMap& GetRunningColumnFamiliesTimestampSize() + const { return running_ts_sz_; } - const std::unordered_map& + const UnorderedMap& GetColumnFamiliesTimestampSizeForRecord() const { return ts_sz_for_record_; } @@ -744,10 +744,10 @@ class ColumnFamilySet { // the same requirements as `column_families_` and `column_family_data_`. // Mapping from column family id to user-defined timestamp size for all // running column families. - std::unordered_map running_ts_sz_; + UnorderedMap running_ts_sz_; // Mapping from column family id to user-defined timestamp size for // column families with non-zero user-defined timestamp size. - std::unordered_map ts_sz_for_record_; + UnorderedMap ts_sz_for_record_; uint32_t max_column_family_; const FileOptions file_options_; diff --git a/db/db_impl/db_impl_open.cc b/db/db_impl/db_impl_open.cc index 931062298d16e84e218795897b07a65132e97060..2e0c91aba23ef48858d660ec7af4632ee03d5843 100644 --- a/db/db_impl/db_impl_open.cc +++ b/db/db_impl/db_impl_open.cc @@ -1187,7 +1187,7 @@ Status DBImpl::RecoverLogFiles(const std::vector& wal_numbers, std::string scratch; Slice record; - const std::unordered_map& running_ts_sz = + const UnorderedMap& running_ts_sz = versions_->GetRunningColumnFamiliesTimestampSize(); TEST_SYNC_POINT_CALLBACK("DBImpl::RecoverLogFiles:BeforeReadWal", @@ -1213,7 +1213,7 @@ Status DBImpl::RecoverLogFiles(const std::vector& wal_numbers, return status; } - const std::unordered_map& record_ts_sz = + const UnorderedMap& record_ts_sz = reader.GetRecordedTimestampSize(); // TODO(yuzhangyu): update mode to kReconcileInconsistency when user // comparator can be changed. diff --git a/db/db_impl/db_impl_secondary.cc b/db/db_impl/db_impl_secondary.cc index 3103428976340528fb430f38672ab92b2f5b440f..de3536841c43664a56e11d7641723634ac117a31 100644 --- a/db/db_impl/db_impl_secondary.cc +++ b/db/db_impl/db_impl_secondary.cc @@ -199,7 +199,7 @@ Status DBImplSecondary::RecoverLogFiles( assert(reader != nullptr); } - const std::unordered_map& running_ts_sz = + const UnorderedMap& running_ts_sz = versions_->GetRunningColumnFamiliesTimestampSize(); for (auto log_number : log_numbers) { auto it = log_readers_.find(log_number); @@ -228,7 +228,7 @@ Status DBImplSecondary::RecoverLogFiles( if (!status.ok()) { break; } - const std::unordered_map& record_ts_sz = + const UnorderedMap& record_ts_sz = reader->GetRecordedTimestampSize(); status = HandleWriteBatchTimestampSizeDifference( &batch, running_ts_sz, record_ts_sz, diff --git a/db/log_reader.h b/db/log_reader.h index b7b298e58abe576fb2e6bee4b7ee0d9041ad2072..697d1b5d58c03f4eaee3118d35577e4a67aa63bd 100644 --- a/db/log_reader.h +++ b/db/log_reader.h @@ -20,6 +20,7 @@ #include "rocksdb/slice.h" #include "rocksdb/status.h" #include "util/compression.h" +#include "util/hash_containers.h" #include "util/udt_util.h" #include "util/xxhash.h" @@ -79,7 +80,7 @@ class Reader { // Return the recorded user-defined timestamp size that have been read so // far. This only applies to WAL logs. - const std::unordered_map& GetRecordedTimestampSize() const { + const UnorderedMap& GetRecordedTimestampSize() const { return recorded_cf_to_ts_sz_; } @@ -165,7 +166,7 @@ class Reader { // The recorded user-defined timestamp sizes that have been read so far. This // is only for WAL logs. - std::unordered_map recorded_cf_to_ts_sz_; + UnorderedMap recorded_cf_to_ts_sz_; // Extend record types with the following special values enum { diff --git a/db/log_test.cc b/db/log_test.cc index 7cc4c198e779d99bbd8a7ebf73c4cdca75bb08a7..fa5e2aa0fcd58efc8d6f3ff30fcf223040c8ba17 100644 --- a/db/log_test.cc +++ b/db/log_test.cc @@ -182,9 +182,8 @@ class LogTest Slice* get_reader_contents() { return &reader_contents_; } - void Write( - const std::string& msg, - const std::unordered_map* cf_to_ts_sz = nullptr) { + void Write(const std::string& msg, + const UnorderedMap* cf_to_ts_sz = nullptr) { if (cf_to_ts_sz != nullptr && !cf_to_ts_sz->empty()) { ASSERT_OK(writer_->MaybeAddUserDefinedTimestampSizeRecord(*cf_to_ts_sz)); } @@ -193,10 +192,9 @@ class LogTest size_t WrittenBytes() const { return dest_contents().size(); } - std::string Read( - const WALRecoveryMode wal_recovery_mode = - WALRecoveryMode::kTolerateCorruptedTailRecords, - std::unordered_map* cf_to_ts_sz = nullptr) { + std::string Read(const WALRecoveryMode wal_recovery_mode = + WALRecoveryMode::kTolerateCorruptedTailRecords, + UnorderedMap* cf_to_ts_sz = nullptr) { std::string scratch; Slice record; bool ret = false; @@ -270,9 +268,8 @@ class LogTest } void CheckRecordAndTimestampSize( - std::string record, - std::unordered_map& expected_ts_sz) { - std::unordered_map recorded_ts_sz; + std::string record, UnorderedMap& expected_ts_sz) { + UnorderedMap recorded_ts_sz; ASSERT_EQ(record, Read(WALRecoveryMode:: kTolerateCorruptedTailRecords /* wal_recovery_mode */, @@ -297,18 +294,18 @@ TEST_P(LogTest, ReadWrite) { } TEST_P(LogTest, ReadWriteWithTimestampSize) { - std::unordered_map ts_sz_one = { + UnorderedMap ts_sz_one = { {1, sizeof(uint64_t)}, }; Write("foo", &ts_sz_one); Write("bar"); - std::unordered_map ts_sz_two = {{2, sizeof(char)}}; + UnorderedMap ts_sz_two = {{2, sizeof(char)}}; Write("", &ts_sz_two); Write("xxxx"); CheckRecordAndTimestampSize("foo", ts_sz_one); CheckRecordAndTimestampSize("bar", ts_sz_one); - std::unordered_map expected_ts_sz_two; + UnorderedMap expected_ts_sz_two; // User-defined timestamp size records are accumulated and applied to // subsequent records. expected_ts_sz_two.insert(ts_sz_one.begin(), ts_sz_one.end()); @@ -320,10 +317,9 @@ TEST_P(LogTest, ReadWriteWithTimestampSize) { } TEST_P(LogTest, ReadWriteWithTimestampSizeZeroTimestampIgnored) { - std::unordered_map ts_sz_one = {{1, sizeof(uint64_t)}}; + UnorderedMap ts_sz_one = {{1, sizeof(uint64_t)}}; Write("foo", &ts_sz_one); - std::unordered_map ts_sz_two(ts_sz_one.begin(), - ts_sz_one.end()); + UnorderedMap ts_sz_two(ts_sz_one.begin(), ts_sz_one.end()); ts_sz_two.insert(std::make_pair(2, 0)); Write("bar", &ts_sz_two); @@ -749,7 +745,7 @@ TEST_P(LogTest, RecycleWithTimestampSize) { if (!recyclable_log) { return; // test is only valid for recycled logs } - std::unordered_map ts_sz_one = { + UnorderedMap ts_sz_one = { {1, sizeof(uint32_t)}, }; Write("foo", &ts_sz_one); @@ -765,7 +761,7 @@ TEST_P(LogTest, RecycleWithTimestampSize) { std::unique_ptr dest_holder(new WritableFileWriter( std::move(sink), "" /* don't care */, FileOptions())); Writer recycle_writer(std::move(dest_holder), 123, true); - std::unordered_map ts_sz_two = { + UnorderedMap ts_sz_two = { {2, sizeof(uint64_t)}, }; ASSERT_OK(recycle_writer.MaybeAddUserDefinedTimestampSizeRecord(ts_sz_two)); @@ -1039,18 +1035,18 @@ TEST_P(CompressionLogTest, ReadWriteWithTimestampSize) { return; } ASSERT_OK(SetupTestEnv()); - std::unordered_map ts_sz_one = { + UnorderedMap ts_sz_one = { {1, sizeof(uint64_t)}, }; Write("foo", &ts_sz_one); Write("bar"); - std::unordered_map ts_sz_two = {{2, sizeof(char)}}; + UnorderedMap ts_sz_two = {{2, sizeof(char)}}; Write("", &ts_sz_two); Write("xxxx"); CheckRecordAndTimestampSize("foo", ts_sz_one); CheckRecordAndTimestampSize("bar", ts_sz_one); - std::unordered_map expected_ts_sz_two; + UnorderedMap expected_ts_sz_two; // User-defined timestamp size records are accumulated and applied to // subsequent records. expected_ts_sz_two.insert(ts_sz_one.begin(), ts_sz_one.end()); diff --git a/db/log_writer.cc b/db/log_writer.cc index 1ddf0a26c8855739de28ba4dfcfe6e8d65a24f4e..86e0286ccd57b09cc73f6e1070f19697586eaa23 100644 --- a/db/log_writer.cc +++ b/db/log_writer.cc @@ -197,7 +197,7 @@ IOStatus Writer::AddCompressionTypeRecord() { } IOStatus Writer::MaybeAddUserDefinedTimestampSizeRecord( - const std::unordered_map& cf_to_ts_sz, + const UnorderedMap& cf_to_ts_sz, Env::IOPriority rate_limiter_priority) { std::vector> ts_sz_to_record; for (const auto& [cf_id, ts_sz] : cf_to_ts_sz) { diff --git a/db/log_writer.h b/db/log_writer.h index 75e44d1a4ed89faeed61f8447c3655d339d154e9..7a64a85601587b109c45201e953adc4d53d93fe8 100644 --- a/db/log_writer.h +++ b/db/log_writer.h @@ -20,6 +20,7 @@ #include "rocksdb/slice.h" #include "rocksdb/status.h" #include "util/compression.h" +#include "util/hash_containers.h" namespace ROCKSDB_NAMESPACE { @@ -95,7 +96,7 @@ class Writer { // kRecyclableUserDefinedTimestampSizeType for these column families. // This timestamp size record applies to all subsequent records. IOStatus MaybeAddUserDefinedTimestampSizeRecord( - const std::unordered_map& cf_to_ts_sz, + const UnorderedMap& cf_to_ts_sz, Env::IOPriority rate_limiter_priority = Env::IO_TOTAL); WritableFileWriter* file() { return dest_.get(); } @@ -137,7 +138,7 @@ class Writer { // The recorded user-defined timestamp size that have been written so far. // Since the user-defined timestamp size cannot be changed while the DB is // running, existing entry in this map cannot be updated. - std::unordered_map recorded_cf_to_ts_sz_; + UnorderedMap recorded_cf_to_ts_sz_; }; } // namespace log diff --git a/db/repair.cc b/db/repair.cc index 847e8252459febe5237372a70eb5ac78e55e29bf..8e0b5cf4186afc6438a2bd07a3342c6692b4f4d6 100644 --- a/db/repair.cc +++ b/db/repair.cc @@ -394,7 +394,7 @@ class Repairer { auto cf_mems = new ColumnFamilyMemTablesImpl(vset_.GetColumnFamilySet()); // Read all the records and add to a memtable - const std::unordered_map& running_ts_sz = + const UnorderedMap& running_ts_sz = vset_.GetRunningColumnFamiliesTimestampSize(); std::string scratch; Slice record; @@ -409,7 +409,7 @@ class Repairer { } Status record_status = WriteBatchInternal::SetContents(&batch, record); if (record_status.ok()) { - const std::unordered_map& record_ts_sz = + const UnorderedMap& record_ts_sz = reader.GetRecordedTimestampSize(); record_status = HandleWriteBatchTimestampSizeDifference( &batch, running_ts_sz, record_ts_sz, diff --git a/db/version_set.h b/db/version_set.h index 3993a4a9f32a78ee5f64f1618ac903cd7a4ec7c1..596addc60f756b08750f5c238082b3c51b512bab 100644 --- a/db/version_set.h +++ b/db/version_set.h @@ -1468,12 +1468,12 @@ class VersionSet { ColumnFamilySet* GetColumnFamilySet() { return column_family_set_.get(); } - const std::unordered_map& - GetRunningColumnFamiliesTimestampSize() const { + const UnorderedMap& GetRunningColumnFamiliesTimestampSize() + const { return column_family_set_->GetRunningColumnFamiliesTimestampSize(); } - const std::unordered_map& + const UnorderedMap& GetColumnFamiliesTimestampSizeForRecord() const { return column_family_set_->GetColumnFamiliesTimestampSizeForRecord(); } diff --git a/util/udt_util.cc b/util/udt_util.cc index 6043b7755d9c06df280fd509af9e3c92ab08fc0a..95460e2f454611d5b91f8079b511ee41567975e8 100644 --- a/util/udt_util.cc +++ b/util/udt_util.cc @@ -44,8 +44,8 @@ RecoveryType GetRecoveryType(const size_t running_ts_sz, } bool AllRunningColumnFamiliesConsistent( - const std::unordered_map& running_ts_sz, - const std::unordered_map& record_ts_sz) { + const UnorderedMap& running_ts_sz, + const UnorderedMap& record_ts_sz) { for (const auto& [cf_id, ts_sz] : running_ts_sz) { auto record_it = record_ts_sz.find(cf_id); RecoveryType recovery_type = @@ -61,8 +61,8 @@ bool AllRunningColumnFamiliesConsistent( Status CheckWriteBatchTimestampSizeConsistency( const WriteBatch* batch, - const std::unordered_map& running_ts_sz, - const std::unordered_map& record_ts_sz, + const UnorderedMap& running_ts_sz, + const UnorderedMap& record_ts_sz, TimestampSizeConsistencyMode check_mode, bool* ts_need_recovery) { std::vector column_family_ids; Status status = @@ -103,8 +103,8 @@ Status CheckWriteBatchTimestampSizeConsistency( } // namespace TimestampRecoveryHandler::TimestampRecoveryHandler( - const std::unordered_map& running_ts_sz, - const std::unordered_map& record_ts_sz) + const UnorderedMap& running_ts_sz, + const UnorderedMap& record_ts_sz) : running_ts_sz_(running_ts_sz), record_ts_sz_(record_ts_sz), new_batch_(new WriteBatch()), @@ -234,8 +234,8 @@ Status TimestampRecoveryHandler::ReconcileTimestampDiscrepancy( Status HandleWriteBatchTimestampSizeDifference( const WriteBatch* batch, - const std::unordered_map& running_ts_sz, - const std::unordered_map& record_ts_sz, + const UnorderedMap& running_ts_sz, + const UnorderedMap& record_ts_sz, TimestampSizeConsistencyMode check_mode, std::unique_ptr* new_batch) { // Quick path to bypass checking the WriteBatch. diff --git a/util/udt_util.h b/util/udt_util.h index 7bf8b739eee9bba5b709e8a404270ae831265e0a..fc980a04b41784a2717d6f430fc8b08b9f2bda05 100644 --- a/util/udt_util.h +++ b/util/udt_util.h @@ -16,6 +16,7 @@ #include "rocksdb/status.h" #include "rocksdb/write_batch.h" #include "util/coding.h" +#include "util/hash_containers.h" namespace ROCKSDB_NAMESPACE { @@ -102,9 +103,8 @@ class UserDefinedTimestampSizeRecord { // but not equal, return Status::InvalidArgument. class TimestampRecoveryHandler : public WriteBatch::Handler { public: - TimestampRecoveryHandler( - const std::unordered_map& running_ts_sz, - const std::unordered_map& record_ts_sz); + TimestampRecoveryHandler(const UnorderedMap& running_ts_sz, + const UnorderedMap& record_ts_sz); ~TimestampRecoveryHandler() override {} @@ -155,11 +155,11 @@ class TimestampRecoveryHandler : public WriteBatch::Handler { // Mapping from column family id to user-defined timestamp size for all // running column families including the ones with zero timestamp size. - const std::unordered_map& running_ts_sz_; + const UnorderedMap& running_ts_sz_; // Mapping from column family id to user-defined timestamp size as recorded // in the WAL. This only contains non-zero user-defined timestamp size. - const std::unordered_map& record_ts_sz_; + const UnorderedMap& record_ts_sz_; std::unique_ptr new_batch_; // Handler is valid upon creation and becomes invalid after its `new_batch_` @@ -211,8 +211,8 @@ enum class TimestampSizeConsistencyMode { // families including the ones with zero timestamp size. Status HandleWriteBatchTimestampSizeDifference( const WriteBatch* batch, - const std::unordered_map& running_ts_sz, - const std::unordered_map& record_ts_sz, + const UnorderedMap& running_ts_sz, + const UnorderedMap& record_ts_sz, TimestampSizeConsistencyMode check_mode, std::unique_ptr* new_batch = nullptr); } // namespace ROCKSDB_NAMESPACE diff --git a/util/udt_util_test.cc b/util/udt_util_test.cc index 9c87c23abefa41c69120ab164a7369aae8ca71fe..63f35992a392eec416ecf4d9a04ee7b35ae636e5 100644 --- a/util/udt_util_test.cc +++ b/util/udt_util_test.cc @@ -104,9 +104,8 @@ class HandleTimestampSizeDifferenceTest : public testing::Test { } } - void CreateWriteBatch( - const std::unordered_map& ts_sz_for_batch, - WriteBatch* batch) { + void CreateWriteBatch(const UnorderedMap& ts_sz_for_batch, + WriteBatch* batch) { for (const auto& [cf_id, ts_sz] : ts_sz_for_batch) { std::string key; CreateKey(&key, ts_sz); @@ -185,9 +184,9 @@ class HandleTimestampSizeDifferenceTest : public testing::Test { }; TEST_F(HandleTimestampSizeDifferenceTest, AllColumnFamiliesConsistent) { - std::unordered_map running_ts_sz = {{1, sizeof(uint64_t)}, - {2, 0}}; - std::unordered_map record_ts_sz = {{1, sizeof(uint64_t)}}; + UnorderedMap running_ts_sz = {{1, sizeof(uint64_t)}, + {2, 0}}; + UnorderedMap record_ts_sz = {{1, sizeof(uint64_t)}}; WriteBatch batch; CreateWriteBatch(running_ts_sz, &batch); @@ -204,9 +203,9 @@ TEST_F(HandleTimestampSizeDifferenceTest, AllColumnFamiliesConsistent) { TEST_F(HandleTimestampSizeDifferenceTest, AllInconsistentColumnFamiliesDropped) { - std::unordered_map running_ts_sz = {{2, 0}}; - std::unordered_map record_ts_sz = {{1, sizeof(uint64_t)}, - {3, sizeof(char)}}; + UnorderedMap running_ts_sz = {{2, 0}}; + UnorderedMap record_ts_sz = {{1, sizeof(uint64_t)}, + {3, sizeof(char)}}; WriteBatch batch; CreateWriteBatch(record_ts_sz, &batch); @@ -222,9 +221,9 @@ TEST_F(HandleTimestampSizeDifferenceTest, } TEST_F(HandleTimestampSizeDifferenceTest, InvolvedColumnFamiliesConsistent) { - std::unordered_map running_ts_sz = {{1, sizeof(uint64_t)}, - {2, sizeof(char)}}; - std::unordered_map record_ts_sz = {{1, sizeof(uint64_t)}}; + UnorderedMap running_ts_sz = {{1, sizeof(uint64_t)}, + {2, sizeof(char)}}; + UnorderedMap record_ts_sz = {{1, sizeof(uint64_t)}}; WriteBatch batch; CreateWriteBatch(record_ts_sz, &batch); @@ -241,9 +240,8 @@ TEST_F(HandleTimestampSizeDifferenceTest, InvolvedColumnFamiliesConsistent) { TEST_F(HandleTimestampSizeDifferenceTest, InconsistentColumnFamilyNeedsTimestampStripping) { - std::unordered_map running_ts_sz = {{1, 0}, - {2, sizeof(char)}}; - std::unordered_map record_ts_sz = {{1, sizeof(uint64_t)}}; + UnorderedMap running_ts_sz = {{1, 0}, {2, sizeof(char)}}; + UnorderedMap record_ts_sz = {{1, sizeof(uint64_t)}}; WriteBatch batch; CreateWriteBatch(record_ts_sz, &batch); @@ -265,10 +263,10 @@ TEST_F(HandleTimestampSizeDifferenceTest, TEST_F(HandleTimestampSizeDifferenceTest, InconsistentColumnFamilyNeedsTimestampPadding) { - std::unordered_map running_ts_sz = {{1, sizeof(uint64_t)}}; + UnorderedMap running_ts_sz = {{1, sizeof(uint64_t)}}; // Make `record_ts_sz` not contain zero timestamp size entries to follow the // behavior of actual WAL log timestamp size record. - std::unordered_map record_ts_sz; + UnorderedMap record_ts_sz; WriteBatch batch; CreateWriteBatch({{1, 0}}, &batch); @@ -289,9 +287,9 @@ TEST_F(HandleTimestampSizeDifferenceTest, TEST_F(HandleTimestampSizeDifferenceTest, InconsistencyReconcileCopyOverDroppedColumnFamily) { - std::unordered_map running_ts_sz = {{1, 0}}; - std::unordered_map record_ts_sz = {{1, sizeof(uint64_t)}, - {2, sizeof(char)}}; + UnorderedMap running_ts_sz = {{1, 0}}; + UnorderedMap record_ts_sz = {{1, sizeof(uint64_t)}, + {2, sizeof(char)}}; WriteBatch batch; CreateWriteBatch(record_ts_sz, &batch); std::unique_ptr new_batch(nullptr); @@ -308,8 +306,8 @@ TEST_F(HandleTimestampSizeDifferenceTest, } TEST_F(HandleTimestampSizeDifferenceTest, UnrecoverableInconsistency) { - std::unordered_map running_ts_sz = {{1, sizeof(char)}}; - std::unordered_map record_ts_sz = {{1, sizeof(uint64_t)}}; + UnorderedMap running_ts_sz = {{1, sizeof(char)}}; + UnorderedMap record_ts_sz = {{1, sizeof(uint64_t)}}; WriteBatch batch; CreateWriteBatch(record_ts_sz, &batch);