From 397ac0fb245dd2aaa2db9d5818f3a7aa09ba2f56 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 13 Jun 2018 05:52:03 +0300 Subject: [PATCH] Updated performance testing tool #1890 --- dbms/src/Common/ErrorCodes.cpp | 1 + dbms/src/IO/MMapReadBufferFromFile.cpp | 72 +++++++++++++++ dbms/src/IO/MMapReadBufferFromFile.h | 37 ++++++++ .../IO/MMapReadBufferFromFileDescriptor.cpp | 90 +++++++++++++++++++ .../src/IO/MMapReadBufferFromFileDescriptor.h | 39 ++++++++ dbms/src/IO/ReadBufferFromFileBase.h | 2 +- utils/compressor/decompress_perf.cpp | 4 +- 7 files changed, 243 insertions(+), 2 deletions(-) create mode 100644 dbms/src/IO/MMapReadBufferFromFile.cpp create mode 100644 dbms/src/IO/MMapReadBufferFromFile.h create mode 100644 dbms/src/IO/MMapReadBufferFromFileDescriptor.cpp create mode 100644 dbms/src/IO/MMapReadBufferFromFileDescriptor.h diff --git a/dbms/src/Common/ErrorCodes.cpp b/dbms/src/Common/ErrorCodes.cpp index 79301034ab..21e5f65cda 100644 --- a/dbms/src/Common/ErrorCodes.cpp +++ b/dbms/src/Common/ErrorCodes.cpp @@ -374,6 +374,7 @@ namespace ErrorCodes extern const int QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW = 397; extern const int UNKNOWN_MUTATION_COMMAND = 398; extern const int FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT = 399; + extern const int CANNOT_STAT = 400; extern const int KEEPER_EXCEPTION = 999; diff --git a/dbms/src/IO/MMapReadBufferFromFile.cpp b/dbms/src/IO/MMapReadBufferFromFile.cpp new file mode 100644 index 0000000000..7596801330 --- /dev/null +++ b/dbms/src/IO/MMapReadBufferFromFile.cpp @@ -0,0 +1,72 @@ +#include +#include + +#include +#include +#include + + +namespace ProfileEvents +{ + extern const Event FileOpen; + extern const Event FileOpenFailed; +} + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int FILE_DOESNT_EXIST; + extern const int CANNOT_OPEN_FILE; + extern const int CANNOT_CLOSE_FILE; +} + + +void MMapReadBufferFromFile::open(const std::string & file_name) +{ + ProfileEvents::increment(ProfileEvents::FileOpen); + + fd = ::open(file_name.c_str(), O_RDONLY); + + if (-1 == fd) + { + ProfileEvents::increment(ProfileEvents::FileOpenFailed); + throwFromErrno("Cannot open file " + file_name, errno == ENOENT ? ErrorCodes::FILE_DOESNT_EXIST : ErrorCodes::CANNOT_OPEN_FILE); + } +} + + +MMapReadBufferFromFile::MMapReadBufferFromFile(const std::string & file_name, size_t offset, size_t length) +{ + open(file_name); + init(fd, offset, length); +} + + +MMapReadBufferFromFile::MMapReadBufferFromFile(const std::string & file_name, size_t offset) +{ + open(file_name); + init(fd, offset); +} + + +MMapReadBufferFromFile::~MMapReadBufferFromFile() +{ + if (fd != -1) + close(); /// Exceptions will lead to std::terminate and that's Ok. +} + + +void MMapReadBufferFromFile::close() +{ + finish(); + + if (0 != ::close(fd)) + throw Exception("Cannot close file", ErrorCodes::CANNOT_CLOSE_FILE); + + fd = -1; + metric_increment.destroy(); +} + +} diff --git a/dbms/src/IO/MMapReadBufferFromFile.h b/dbms/src/IO/MMapReadBufferFromFile.h new file mode 100644 index 0000000000..c1762bd54f --- /dev/null +++ b/dbms/src/IO/MMapReadBufferFromFile.h @@ -0,0 +1,37 @@ +#pragma once + +#include +#include + + +namespace CurrentMetrics +{ + extern const Metric OpenFileForRead; +} + + +namespace DB +{ + +class MMapReadBufferFromFile : public MMapReadBufferFromFileDescriptor +{ +public: + MMapReadBufferFromFile(const std::string & file_name, size_t offset, size_t length); + + /// Map till end of file. + MMapReadBufferFromFile(const std::string & file_name, size_t offset); + + ~MMapReadBufferFromFile() override; + + void close(); + +private: + int fd = -1; + + CurrentMetrics::Increment metric_increment{CurrentMetrics::OpenFileForRead}; + + void open(const std::string & file_name); +}; + +} + diff --git a/dbms/src/IO/MMapReadBufferFromFileDescriptor.cpp b/dbms/src/IO/MMapReadBufferFromFileDescriptor.cpp new file mode 100644 index 0000000000..4643b9b626 --- /dev/null +++ b/dbms/src/IO/MMapReadBufferFromFileDescriptor.cpp @@ -0,0 +1,90 @@ +#include +#include +#include +#include + +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int CANNOT_ALLOCATE_MEMORY; + extern const int CANNOT_MUNMAP; + extern const int CANNOT_STAT; + extern const int BAD_ARGUMENTS; + extern const int LOGICAL_ERROR; +} + + +void MMapReadBufferFromFileDescriptor::init(int fd_, size_t offset, size_t length_) +{ + fd = fd_; + length = length_; + + if (length) + { + void * buf = mmap(nullptr, length, PROT_READ, MAP_PRIVATE, fd, offset); + if (MAP_FAILED == buf) + throwFromErrno("MMapReadBufferFromFileDescriptor: Cannot mmap " + formatReadableSizeWithBinarySuffix(length) + ".", + ErrorCodes::CANNOT_ALLOCATE_MEMORY); + + BufferBase::set(static_cast(buf), length, 0); + } +} + +void MMapReadBufferFromFileDescriptor::init(int fd_, size_t offset) +{ + fd = fd_; + + struct stat stat_res {}; + if (0 != fstat(fd, &stat_res)) + throwFromErrno("MMapReadBufferFromFileDescriptor: Cannot fstat.", ErrorCodes::CANNOT_STAT); + + off_t file_size = stat_res.st_size; + + if (file_size < 0) + throw Exception("MMapReadBufferFromFileDescriptor: fstat returned negative file size", ErrorCodes::LOGICAL_ERROR); + + if (offset > static_cast(file_size)) + throw Exception("MMapReadBufferFromFileDescriptor: requested offset is greater than file size", ErrorCodes::BAD_ARGUMENTS); + + init(fd, offset, file_size - offset); +} + + +MMapReadBufferFromFileDescriptor::MMapReadBufferFromFileDescriptor(int fd, size_t offset, size_t length) + : MMapReadBufferFromFileDescriptor() +{ + init(fd, offset, length); +} + + +MMapReadBufferFromFileDescriptor::MMapReadBufferFromFileDescriptor(int fd, size_t offset) + : MMapReadBufferFromFileDescriptor() +{ + init(fd, offset); +} + + +MMapReadBufferFromFileDescriptor::~MMapReadBufferFromFileDescriptor() +{ + if (length) + finish(); /// Exceptions will lead to std::terminate and that's Ok. +} + + +void MMapReadBufferFromFileDescriptor::finish() +{ + if (0 != munmap(internalBuffer().begin(), length)) + throwFromErrno("MMapReadBufferFromFileDescriptor: Cannot munmap " + formatReadableSizeWithBinarySuffix(length) + ".", + ErrorCodes::CANNOT_MUNMAP); + + length = 0; +} + +} diff --git a/dbms/src/IO/MMapReadBufferFromFileDescriptor.h b/dbms/src/IO/MMapReadBufferFromFileDescriptor.h new file mode 100644 index 0000000000..90bfe640df --- /dev/null +++ b/dbms/src/IO/MMapReadBufferFromFileDescriptor.h @@ -0,0 +1,39 @@ +#pragma once + +#include + + +namespace DB +{ + +/** MMap range in a file and represent it as a ReadBuffer. + * Please note that mmap is not always the optimal way to read file. + * Also you cannot control whether and how long actual IO take place, + * so this method is not manageable and not recommended for anything except benchmarks. + */ +class MMapReadBufferFromFileDescriptor : public ReadBuffer +{ +protected: + MMapReadBufferFromFileDescriptor() : ReadBuffer(nullptr, 0) {}; + + void init(int fd_, size_t offset, size_t length_); + void init(int fd_, size_t offset); + +public: + MMapReadBufferFromFileDescriptor(int fd, size_t offset, size_t length); + + /// Map till end of file. + MMapReadBufferFromFileDescriptor(int fd, size_t offset); + + ~MMapReadBufferFromFileDescriptor() override; + + /// unmap memory before call to destructor + void finish(); + +private: + size_t length = 0; + int fd = -1; +}; + +} + diff --git a/dbms/src/IO/ReadBufferFromFileBase.h b/dbms/src/IO/ReadBufferFromFileBase.h index dbbc667edf..bdadc0ffe6 100644 --- a/dbms/src/IO/ReadBufferFromFileBase.h +++ b/dbms/src/IO/ReadBufferFromFileBase.h @@ -19,7 +19,7 @@ class ReadBufferFromFileBase : public BufferWithOwnMemory public: ReadBufferFromFileBase(size_t buf_size, char * existing_memory, size_t alignment); ReadBufferFromFileBase(ReadBufferFromFileBase &&) = default; - virtual ~ReadBufferFromFileBase(); + ~ReadBufferFromFileBase() override; off_t seek(off_t off, int whence = SEEK_SET); virtual off_t getPositionInFile() = 0; virtual std::string getFileName() const = 0; diff --git a/utils/compressor/decompress_perf.cpp b/utils/compressor/decompress_perf.cpp index 0c284c5e37..e6234980b7 100644 --- a/utils/compressor/decompress_perf.cpp +++ b/utils/compressor/decompress_perf.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -172,7 +173,8 @@ try */ ssize_t variant = argc < 2 ? -1 : parse(argv[1]); - ReadBufferFromFileDescriptor in(STDIN_FILENO); + MMapReadBufferFromFileDescriptor in(STDIN_FILENO, 0); +// ReadBufferFromFileDescriptor in(STDIN_FILENO); FasterCompressedReadBuffer decompressing_in(in, variant); // WriteBufferFromFileDescriptor out(STDOUT_FILENO); // HashingWriteBuffer hashing_out(out); -- GitLab