From 524c6e6b7290dc4aaf130ce035a199d4b9d6dc7d Mon Sep 17 00:00:00 2001 From: Yanqin Jin Date: Thu, 21 Jun 2018 08:34:24 -0700 Subject: [PATCH] Add file name info to SequentialFileReader. (#4026) Summary: We potentially need this information for tracing, profiling and diagnosis. Closes https://github.com/facebook/rocksdb/pull/4026 Differential Revision: D8555214 Pulled By: riversand963 fbshipit-source-id: 4263e06c00b6d5410b46aa46eb4e358ff2161dd2 --- db/db_impl.cc | 3 ++- db/db_impl_open.cc | 2 +- db/log_test.cc | 12 ++++++------ db/repair.cc | 2 +- db/transaction_log_impl.cc | 7 ++++--- db/version_set.cc | 6 +++--- db/wal_manager.cc | 2 +- port/win/win_jemalloc.cc | 6 ++---- tools/ldb_cmd.cc | 3 ++- util/file_reader_writer.h | 8 ++++++-- util/file_util.cc | 2 +- util/testutil.cc | 5 +++-- util/testutil.h | 3 ++- utilities/backupable/backupable_db.cc | 6 +++--- utilities/blob_db/blob_file.cc | 5 +++-- 15 files changed, 40 insertions(+), 32 deletions(-) diff --git a/db/db_impl.cc b/db/db_impl.cc index 30318860e..84c9dca75 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -2368,7 +2368,8 @@ Status DBImpl::GetDbIdentity(std::string& identity) const { if (!s.ok()) { return s; } - id_file_reader.reset(new SequentialFileReader(std::move(idfile))); + id_file_reader.reset( + new SequentialFileReader(std::move(idfile), idfilename)); } uint64_t file_size; diff --git a/db/db_impl_open.cc b/db/db_impl_open.cc index 2ab347893..48753723f 100644 --- a/db/db_impl_open.cc +++ b/db/db_impl_open.cc @@ -577,7 +577,7 @@ Status DBImpl::RecoverLogFiles(const std::vector& log_numbers, continue; } } - file_reader.reset(new SequentialFileReader(std::move(file))); + file_reader.reset(new SequentialFileReader(std::move(file), fname)); } // Create the log reader. diff --git a/db/log_test.cc b/db/log_test.cc index ee79e10af..992851fe2 100644 --- a/db/log_test.cc +++ b/db/log_test.cc @@ -160,8 +160,8 @@ class LogTest : public ::testing::TestWithParam { : reader_contents_(), dest_holder_(test::GetWritableFileWriter( new test::StringSink(&reader_contents_))), - source_holder_( - test::GetSequentialFileReader(new StringSource(reader_contents_))), + source_holder_(test::GetSequentialFileReader( + new StringSource(reader_contents_), "" /* file name */)), writer_(std::move(dest_holder_), 123, GetParam()), reader_(nullptr, std::move(source_holder_), &report_, true /*checksum*/, 0 /*initial_offset*/, 123) { @@ -268,8 +268,8 @@ class LogTest : public ::testing::TestWithParam { void CheckOffsetPastEndReturnsNoRecords(uint64_t offset_past_end) { WriteInitialOffsetLog(); - unique_ptr file_reader( - test::GetSequentialFileReader(new StringSource(reader_contents_))); + unique_ptr file_reader(test::GetSequentialFileReader( + new StringSource(reader_contents_), "" /* fname */)); unique_ptr offset_reader( new Reader(nullptr, std::move(file_reader), &report_, true /*checksum*/, WrittenBytes() + offset_past_end, 123)); @@ -281,8 +281,8 @@ class LogTest : public ::testing::TestWithParam { void CheckInitialOffsetRecord(uint64_t initial_offset, int expected_record_offset) { WriteInitialOffsetLog(); - unique_ptr file_reader( - test::GetSequentialFileReader(new StringSource(reader_contents_))); + unique_ptr file_reader(test::GetSequentialFileReader( + new StringSource(reader_contents_), "" /* fname */)); unique_ptr offset_reader( new Reader(nullptr, std::move(file_reader), &report_, true /*checksum*/, initial_offset, 123)); diff --git a/db/repair.cc b/db/repair.cc index 4e41960ff..b10873504 100644 --- a/db/repair.cc +++ b/db/repair.cc @@ -341,7 +341,7 @@ class Repairer { return status; } unique_ptr lfile_reader( - new SequentialFileReader(std::move(lfile))); + new SequentialFileReader(std::move(lfile), logname)); // Create the log reader. LogReporter reporter; diff --git a/db/transaction_log_impl.cc b/db/transaction_log_impl.cc index 011673892..dce1b093f 100644 --- a/db/transaction_log_impl.cc +++ b/db/transaction_log_impl.cc @@ -46,13 +46,14 @@ Status TransactionLogIteratorImpl::OpenLogFile( const LogFile* logFile, unique_ptr* file_reader) { Env* env = options_->env; unique_ptr file; + std::string fname; Status s; EnvOptions optimized_env_options = env->OptimizeForLogRead(soptions_); if (logFile->Type() == kArchivedLogFile) { - std::string fname = ArchivedLogFileName(dir_, logFile->LogNumber()); + fname = ArchivedLogFileName(dir_, logFile->LogNumber()); s = env->NewSequentialFile(fname, &file, optimized_env_options); } else { - std::string fname = LogFileName(dir_, logFile->LogNumber()); + fname = LogFileName(dir_, logFile->LogNumber()); s = env->NewSequentialFile(fname, &file, optimized_env_options); if (!s.ok()) { // If cannot open file in DB directory. @@ -62,7 +63,7 @@ Status TransactionLogIteratorImpl::OpenLogFile( } } if (s.ok()) { - file_reader->reset(new SequentialFileReader(std::move(file))); + file_reader->reset(new SequentialFileReader(std::move(file), fname)); } return s; } diff --git a/db/version_set.cc b/db/version_set.cc index 02dc29402..23b4bb4cd 100644 --- a/db/version_set.cc +++ b/db/version_set.cc @@ -3106,7 +3106,7 @@ Status VersionSet::Recover( return s; } manifest_file_reader.reset( - new SequentialFileReader(std::move(manifest_file))); + new SequentialFileReader(std::move(manifest_file), manifest_filename)); } uint64_t current_manifest_file_size; s = env_->GetFileSize(manifest_filename, ¤t_manifest_file_size); @@ -3416,7 +3416,7 @@ Status VersionSet::ListColumnFamilies(std::vector* column_families, if (!s.ok()) { return s; } - file_reader.reset(new SequentialFileReader(std::move(file))); + file_reader.reset(new SequentialFileReader(std::move(file), dscname)); } std::map column_family_names; @@ -3560,7 +3560,7 @@ Status VersionSet::DumpManifest(Options& options, std::string& dscname, if (!s.ok()) { return s; } - file_reader.reset(new SequentialFileReader(std::move(file))); + file_reader.reset(new SequentialFileReader(std::move(file), dscname)); } bool have_prev_log_number = false; diff --git a/db/wal_manager.cc b/db/wal_manager.cc index 770470f6e..3b0ef7ca5 100644 --- a/db/wal_manager.cc +++ b/db/wal_manager.cc @@ -444,7 +444,7 @@ Status WalManager::ReadFirstLine(const std::string& fname, Status status = env_->NewSequentialFile( fname, &file, env_->OptimizeForLogRead(env_options_)); unique_ptr file_reader( - new SequentialFileReader(std::move(file))); + new SequentialFileReader(std::move(file), fname)); if (!status.ok()) { return status; diff --git a/port/win/win_jemalloc.cc b/port/win/win_jemalloc.cc index 18caf2cba..6e5bfd427 100644 --- a/port/win/win_jemalloc.cc +++ b/port/win/win_jemalloc.cc @@ -37,12 +37,10 @@ ZSTD_customMem GetJeZstdAllocationOverrides() { // Global operators to be replaced by a linker when this file is // a part of the build -void* jemalloc_aligned_alloc( size_t size, size_t alignment) ROCKSDB_NOEXCEPT { +void* jemalloc_aligned_alloc(size_t size, size_t alignment) ROCKSDB_NOEXCEPT { return je_aligned_alloc(alignment, size); } -void jemalloc_aligned_free(void* p) ROCKSDB_NOEXCEPT { - je_free(p); -} +void jemalloc_aligned_free(void* p) ROCKSDB_NOEXCEPT { je_free(p); } void* operator new(size_t size) { void* p = je_malloc(size); diff --git a/tools/ldb_cmd.cc b/tools/ldb_cmd.cc index 330ce00b0..62c3a6af9 100644 --- a/tools/ldb_cmd.cc +++ b/tools/ldb_cmd.cc @@ -1962,7 +1962,8 @@ void DumpWalFile(std::string wal_file, bool print_header, bool print_values, unique_ptr file; status = env_->NewSequentialFile(wal_file, &file, soptions); if (status.ok()) { - wal_file_reader.reset(new SequentialFileReader(std::move(file))); + wal_file_reader.reset( + new SequentialFileReader(std::move(file), wal_file)); } } if (!status.ok()) { diff --git a/util/file_reader_writer.h b/util/file_reader_writer.h index 6d62e54ca..4d3dbb1c3 100644 --- a/util/file_reader_writer.h +++ b/util/file_reader_writer.h @@ -26,11 +26,13 @@ std::unique_ptr NewReadaheadRandomAccessFile( class SequentialFileReader { private: std::unique_ptr file_; + std::string file_name_; std::atomic offset_; // read offset public: - explicit SequentialFileReader(std::unique_ptr&& _file) - : file_(std::move(_file)), offset_(0) {} + explicit SequentialFileReader(std::unique_ptr&& _file, + const std::string& _file_name) + : file_(std::move(_file)), file_name_(_file_name), offset_(0) {} SequentialFileReader(SequentialFileReader&& o) ROCKSDB_NOEXCEPT { *this = std::move(o); @@ -52,6 +54,8 @@ class SequentialFileReader { SequentialFile* file() { return file_.get(); } + std::string file_name() { return file_name_; } + bool use_direct_io() const { return file_->use_direct_io(); } }; diff --git a/util/file_util.cc b/util/file_util.cc index 195fe2ec4..ee8b3fb53 100644 --- a/util/file_util.cc +++ b/util/file_util.cc @@ -41,7 +41,7 @@ Status CopyFile(Env* env, const std::string& source, return s; } } - src_reader.reset(new SequentialFileReader(std::move(srcfile))); + src_reader.reset(new SequentialFileReader(std::move(srcfile), source)); dest_writer.reset(new WritableFileWriter(std::move(destfile), soptions)); } diff --git a/util/testutil.cc b/util/testutil.cc index 79bffcc1d..fa32498a2 100644 --- a/util/testutil.cc +++ b/util/testutil.cc @@ -135,9 +135,10 @@ RandomAccessFileReader* GetRandomAccessFileReader(RandomAccessFile* raf) { "[test RandomAccessFileReader]"); } -SequentialFileReader* GetSequentialFileReader(SequentialFile* se) { +SequentialFileReader* GetSequentialFileReader(SequentialFile* se, + const std::string& fname) { unique_ptr file(se); - return new SequentialFileReader(std::move(file)); + return new SequentialFileReader(std::move(file), fname); } void CorruptKeyType(InternalKey* ikey) { diff --git a/util/testutil.h b/util/testutil.h index de181e002..d0440b1a8 100644 --- a/util/testutil.h +++ b/util/testutil.h @@ -187,7 +187,8 @@ extern WritableFileWriter* GetWritableFileWriter(WritableFile* wf); extern RandomAccessFileReader* GetRandomAccessFileReader(RandomAccessFile* raf); -extern SequentialFileReader* GetSequentialFileReader(SequentialFile* se); +extern SequentialFileReader* GetSequentialFileReader(SequentialFile* se, + const std::string& fname); class StringSink: public WritableFile { public: diff --git a/utilities/backupable/backupable_db.cc b/utilities/backupable/backupable_db.cc index d911d6fba..a7fa08268 100644 --- a/utilities/backupable/backupable_db.cc +++ b/utilities/backupable/backupable_db.cc @@ -1218,7 +1218,7 @@ Status BackupEngineImpl::CopyOrCreateFile( unique_ptr src_reader; unique_ptr buf; if (!src.empty()) { - src_reader.reset(new SequentialFileReader(std::move(src_file))); + src_reader.reset(new SequentialFileReader(std::move(src_file), src)); buf.reset(new char[copy_file_buffer_size_]); } @@ -1417,7 +1417,7 @@ Status BackupEngineImpl::CalculateChecksum(const std::string& src, Env* src_env, } unique_ptr src_reader( - new SequentialFileReader(std::move(src_file))); + new SequentialFileReader(std::move(src_file), src)); std::unique_ptr buf(new char[copy_file_buffer_size_]); Slice data; @@ -1641,7 +1641,7 @@ Status BackupEngineImpl::BackupMeta::LoadFromFile( } unique_ptr backup_meta_reader( - new SequentialFileReader(std::move(backup_meta_file))); + new SequentialFileReader(std::move(backup_meta_file), meta_filename_)); unique_ptr buf(new char[max_backup_meta_file_size_ + 1]); Slice data; s = backup_meta_reader->Read(max_backup_meta_file_size_, &data, buf.get()); diff --git a/utilities/blob_db/blob_file.cc b/utilities/blob_db/blob_file.cc index 287e62a40..ab4d4efdd 100644 --- a/utilities/blob_db/blob_file.cc +++ b/utilities/blob_db/blob_file.cc @@ -84,14 +84,15 @@ std::shared_ptr BlobFile::OpenSequentialReader( Env* env, const DBOptions& db_options, const EnvOptions& env_options) const { std::unique_ptr sfile; - Status s = env->NewSequentialFile(PathName(), &sfile, env_options); + std::string path_name(PathName()); + Status s = env->NewSequentialFile(path_name, &sfile, env_options); if (!s.ok()) { // report something here. return nullptr; } std::unique_ptr sfile_reader; - sfile_reader.reset(new SequentialFileReader(std::move(sfile))); + sfile_reader.reset(new SequentialFileReader(std::move(sfile), path_name)); std::shared_ptr log_reader = std::make_shared( std::move(sfile_reader), db_options.env, db_options.statistics.get()); -- GitLab