Updated performance testing tool #1890

dbms/src/Common/ErrorCodes.cpp

@@ -374,6 +374,7 @@ namespace ErrorCodes
     extern const int QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW = 397;
     extern const int UNKNOWN_MUTATION_COMMAND = 398;
     extern const int FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT = 399;
+    extern const int CANNOT_STAT = 400;
 
 
     extern const int KEEPER_EXCEPTION = 999;

dbms/src/IO/MMapReadBufferFromFile.cpp

+#include <unistd.h>
+#include <fcntl.h>
+
+#include <Common/ProfileEvents.h>
+#include <Common/formatReadable.h>
+#include <IO/MMapReadBufferFromFile.h>
+
+
+namespace ProfileEvents
+{
+    extern const Event FileOpen;
+    extern const Event FileOpenFailed;
+}
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int FILE_DOESNT_EXIST;
+    extern const int CANNOT_OPEN_FILE;
+    extern const int CANNOT_CLOSE_FILE;
+}
+
+
+void MMapReadBufferFromFile::open(const std::string & file_name)
+{
+    ProfileEvents::increment(ProfileEvents::FileOpen);
+
+    fd = ::open(file_name.c_str(), O_RDONLY);
+
+    if (-1 == fd)
+    {
+        ProfileEvents::increment(ProfileEvents::FileOpenFailed);
+        throwFromErrno("Cannot open file " + file_name, errno == ENOENT ? ErrorCodes::FILE_DOESNT_EXIST : ErrorCodes::CANNOT_OPEN_FILE);
+    }
+}
+
+
+MMapReadBufferFromFile::MMapReadBufferFromFile(const std::string & file_name, size_t offset, size_t length)
+{
+    open(file_name);
+    init(fd, offset, length);
+}
+
+
+MMapReadBufferFromFile::MMapReadBufferFromFile(const std::string & file_name, size_t offset)
+{
+    open(file_name);
+    init(fd, offset);
+}
+
+
+MMapReadBufferFromFile::~MMapReadBufferFromFile()
+{
+    if (fd != -1)
+        close();    /// Exceptions will lead to std::terminate and that's Ok.
+}
+
+
+void MMapReadBufferFromFile::close()
+{
+    finish();
+
+    if (0 != ::close(fd))
+        throw Exception("Cannot close file", ErrorCodes::CANNOT_CLOSE_FILE);
+
+    fd = -1;
+    metric_increment.destroy();
+}
+
+}

dbms/src/IO/MMapReadBufferFromFile.h

+#pragma once
+
+#include <Common/CurrentMetrics.h>
+#include <IO/MMapReadBufferFromFileDescriptor.h>
+
+
+namespace CurrentMetrics
+{
+    extern const Metric OpenFileForRead;
+}
+
+
+namespace DB
+{
+
+class MMapReadBufferFromFile : public MMapReadBufferFromFileDescriptor
+{
+public:
+    MMapReadBufferFromFile(const std::string & file_name, size_t offset, size_t length);
+
+    /// Map till end of file.
+    MMapReadBufferFromFile(const std::string & file_name, size_t offset);
+
+    ~MMapReadBufferFromFile() override;
+
+    void close();
+
+private:
+    int fd = -1;
+
+    CurrentMetrics::Increment metric_increment{CurrentMetrics::OpenFileForRead};
+
+    void open(const std::string & file_name);
+};
+
+}
+

dbms/src/IO/MMapReadBufferFromFileDescriptor.cpp

+#include <sys/mman.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include <Common/ProfileEvents.h>
+#include <Common/formatReadable.h>
+#include <IO/MMapReadBufferFromFileDescriptor.h>
+
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int CANNOT_ALLOCATE_MEMORY;
+    extern const int CANNOT_MUNMAP;
+    extern const int CANNOT_STAT;
+    extern const int BAD_ARGUMENTS;
+    extern const int LOGICAL_ERROR;
+}
+
+
+void MMapReadBufferFromFileDescriptor::init(int fd_, size_t offset, size_t length_)
+{
+    fd = fd_;
+    length = length_;
+
+    if (length)
+    {
+        void * buf = mmap(nullptr, length, PROT_READ, MAP_PRIVATE, fd, offset);
+        if (MAP_FAILED == buf)
+            throwFromErrno("MMapReadBufferFromFileDescriptor: Cannot mmap " + formatReadableSizeWithBinarySuffix(length) + ".",
+                ErrorCodes::CANNOT_ALLOCATE_MEMORY);
+
+        BufferBase::set(static_cast<char *>(buf), length, 0);
+    }
+}
+
+void MMapReadBufferFromFileDescriptor::init(int fd_, size_t offset)
+{
+    fd = fd_;
+
+    struct stat stat_res {};
+    if (0 != fstat(fd, &stat_res))
+        throwFromErrno("MMapReadBufferFromFileDescriptor: Cannot fstat.", ErrorCodes::CANNOT_STAT);
+
+    off_t file_size = stat_res.st_size;
+
+    if (file_size < 0)
+        throw Exception("MMapReadBufferFromFileDescriptor: fstat returned negative file size", ErrorCodes::LOGICAL_ERROR);
+
+    if (offset > static_cast<size_t>(file_size))
+        throw Exception("MMapReadBufferFromFileDescriptor: requested offset is greater than file size", ErrorCodes::BAD_ARGUMENTS);
+
+    init(fd, offset, file_size - offset);
+}
+
+
+MMapReadBufferFromFileDescriptor::MMapReadBufferFromFileDescriptor(int fd, size_t offset, size_t length)
+    : MMapReadBufferFromFileDescriptor()
+{
+    init(fd, offset, length);
+}
+
+
+MMapReadBufferFromFileDescriptor::MMapReadBufferFromFileDescriptor(int fd, size_t offset)
+    : MMapReadBufferFromFileDescriptor()
+{
+    init(fd, offset);
+}
+
+
+MMapReadBufferFromFileDescriptor::~MMapReadBufferFromFileDescriptor()
+{
+    if (length)
+        finish();    /// Exceptions will lead to std::terminate and that's Ok.
+}
+
+
+void MMapReadBufferFromFileDescriptor::finish()
+{
+    if (0 != munmap(internalBuffer().begin(), length))
+        throwFromErrno("MMapReadBufferFromFileDescriptor: Cannot munmap " + formatReadableSizeWithBinarySuffix(length) + ".",
+            ErrorCodes::CANNOT_MUNMAP);
+
+    length = 0;
+}
+
+}

dbms/src/IO/MMapReadBufferFromFileDescriptor.h

+#pragma once
+
+#include <IO/ReadBuffer.h>
+
+
+namespace DB
+{
+
+/** MMap range in a file and represent it as a ReadBuffer.
+  * Please note that mmap is not always the optimal way to read file.
+  * Also you cannot control whether and how long actual IO take place,
+  *  so this method is not manageable and not recommended for anything except benchmarks.
+  */
+class MMapReadBufferFromFileDescriptor : public ReadBuffer
+{
+protected:
+    MMapReadBufferFromFileDescriptor() : ReadBuffer(nullptr, 0) {};
+
+    void init(int fd_, size_t offset, size_t length_);
+    void init(int fd_, size_t offset);
+
+public:
+    MMapReadBufferFromFileDescriptor(int fd, size_t offset, size_t length);
+
+    /// Map till end of file.
+    MMapReadBufferFromFileDescriptor(int fd, size_t offset);
+
+    ~MMapReadBufferFromFileDescriptor() override;
+
+    /// unmap memory before call to destructor
+    void finish();
+
+private:
+    size_t length = 0;
+    int fd = -1;
+};
+
+}
+

dbms/src/IO/ReadBufferFromFileBase.h

@@ -19,7 +19,7 @@ class ReadBufferFromFileBase : public BufferWithOwnMemory<ReadBuffer>
 public:
     ReadBufferFromFileBase(size_t buf_size, char * existing_memory, size_t alignment);
     ReadBufferFromFileBase(ReadBufferFromFileBase &&) = default;
-    virtual ~ReadBufferFromFileBase();
+    ~ReadBufferFromFileBase() override;
     off_t seek(off_t off, int whence = SEEK_SET);
     virtual off_t getPositionInFile() = 0;
     virtual std::string getFileName() const = 0;

utils/compressor/decompress_perf.cpp

@@ -7,6 +7,7 @@
 #include <IO/ReadBuffer.h>
 #include <IO/ReadBufferFromFileDescriptor.h>
 #include <IO/WriteBufferFromFileDescriptor.h>
+#include <IO/MMapReadBufferFromFileDescriptor.h>
 #include <IO/HashingWriteBuffer.h>
 #include <IO/BufferWithOwnMemory.h>
 #include <IO/CompressedStream.h>
@@ -172,7 +173,8 @@ try
       */
     ssize_t variant = argc < 2 ? -1 : parse<ssize_t>(argv[1]);
 
-    ReadBufferFromFileDescriptor in(STDIN_FILENO);
+    MMapReadBufferFromFileDescriptor in(STDIN_FILENO, 0);
+//    ReadBufferFromFileDescriptor in(STDIN_FILENO);
     FasterCompressedReadBuffer decompressing_in(in, variant);
 //    WriteBufferFromFileDescriptor out(STDOUT_FILENO);
 //    HashingWriteBuffer hashing_out(out);