From 8ef51bf19b1954e8e6591c2e301297b24a4d37ba Mon Sep 17 00:00:00 2001 From: proller Date: Thu, 22 Jun 2017 18:57:37 +0300 Subject: [PATCH] Fixing PerformanceTest: table_exists condition, move getMemoryAmount function to file. Add more debug helpers (#905) * Fix missing includes * ComplexKeyCacheDictionary: Move includes to .cpp * Fixing PerformanceTest: table_exists condition, move getMemoryAmount function to file. Add more debug helpers * Multiplatform memory size getter * Requested changes * Requested changes * Requested changes * fix * Requested changes --- dbms/src/Common/iostream_debug_helpers.cpp | 14 ++- dbms/src/Common/iostream_debug_helpers.h | 3 + .../ComplexKeyCacheDictionary.cpp | 3 + .../Dictionaries/ComplexKeyCacheDictionary.h | 3 - dbms/src/Server/PerformanceTest.cpp | 47 ++++----- libs/libcommon/CMakeLists.txt | 2 + .../include/common/getMemoryAmount.h | 9 ++ .../include/common/iostream_debug_helpers.h | 9 ++ libs/libcommon/src/getMemoryAmount.cpp | 97 +++++++++++++++++++ 9 files changed, 156 insertions(+), 31 deletions(-) create mode 100644 libs/libcommon/include/common/getMemoryAmount.h create mode 100644 libs/libcommon/src/getMemoryAmount.cpp diff --git a/dbms/src/Common/iostream_debug_helpers.cpp b/dbms/src/Common/iostream_debug_helpers.cpp index d4cded2fab..94fb4e211e 100644 --- a/dbms/src/Common/iostream_debug_helpers.cpp +++ b/dbms/src/Common/iostream_debug_helpers.cpp @@ -62,7 +62,7 @@ std::ostream & operator<<(std::ostream & stream, const DB::Block & what) { stream << "Block(" << "size = " << what.getColumns().size() - << ")"; + << "){" << what.dumpStructure() << "}"; return stream; } @@ -79,3 +79,15 @@ std::ostream & operator<<(std::ostream & stream, const DB::IColumn & what) << ")"; return stream; } + +std::ostream & operator<<(std::ostream & stream, const DB::Connection::Packet & what) { + stream << "Connection::Packet(" + << "type = " << what.type; + // types description: Core/Protocol.h + if (what.exception) + stream << "exception = " << what.exception.get(); + //TODO: profile_info + stream << ") {" << what.block << "}"; + return stream; +} + diff --git a/dbms/src/Common/iostream_debug_helpers.h b/dbms/src/Common/iostream_debug_helpers.h index c2cf913273..5f9f7b5692 100644 --- a/dbms/src/Common/iostream_debug_helpers.h +++ b/dbms/src/Common/iostream_debug_helpers.h @@ -32,6 +32,9 @@ std::ostream & operator<<(std::ostream & stream, const DB::ColumnWithTypeAndName namespace DB { class IColumn; } std::ostream & operator<<(std::ostream & stream, const DB::IColumn & what); +#include +std::ostream & operator<<(std::ostream & stream, const DB::Connection::Packet & what); + /// some operator<< should be declared before operator<<(... std::shared_ptr<>) #include diff --git a/dbms/src/Dictionaries/ComplexKeyCacheDictionary.cpp b/dbms/src/Dictionaries/ComplexKeyCacheDictionary.cpp index cc69d5ff93..59f4f2fb8d 100644 --- a/dbms/src/Dictionaries/ComplexKeyCacheDictionary.cpp +++ b/dbms/src/Dictionaries/ComplexKeyCacheDictionary.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -7,6 +8,8 @@ #include #include #include +#include +#include namespace ProfileEvents diff --git a/dbms/src/Dictionaries/ComplexKeyCacheDictionary.h b/dbms/src/Dictionaries/ComplexKeyCacheDictionary.h index f4f1c9a5c9..6c82bc5f8c 100644 --- a/dbms/src/Dictionaries/ComplexKeyCacheDictionary.h +++ b/dbms/src/Dictionaries/ComplexKeyCacheDictionary.h @@ -3,15 +3,12 @@ #include #include #include -#include #include #include #include #include #include -#include #include -#include #include #include #include diff --git a/dbms/src/Server/PerformanceTest.cpp b/dbms/src/Server/PerformanceTest.cpp index af2d2e2adc..d609f4374e 100644 --- a/dbms/src/Server/PerformanceTest.cpp +++ b/dbms/src/Server/PerformanceTest.cpp @@ -2,9 +2,6 @@ #include #include #include -#if __has_include() - #include -#endif #include #include @@ -23,6 +20,7 @@ #include #include #include +#include #include @@ -662,34 +660,24 @@ private: for (const String & precondition : preconditions) { if (precondition == "flush_disk_cache") - if (system("(>&2 echo 'Flushing disk cache...') && (sudo sh -c 'echo 3 > /proc/sys/vm/drop_caches') && (>&2 echo 'Flushed.')")) { + if (system("(>&2 echo 'Flushing disk cache...') && (sudo sh -c 'echo 3 > /proc/sys/vm/drop_caches') && (>&2 echo 'Flushed.')")) + { std::cerr << "Failed to flush disk cache" << std::endl; return false; } if (precondition == "ram_size") { -#if __has_include() - struct sysinfo *system_information = new struct sysinfo(); - if (sysinfo(system_information)) - { - std::cerr << "Failed to check system RAM size" << std::endl; - delete system_information; - } - else + size_t ram_size_needed = config->getUInt64("preconditions.ram_size"); + size_t actual_ram = getMemoryAmount(); + if (!actual_ram) + throw DB::Exception("ram_size precondition not available on this platform", ErrorCodes::NOT_IMPLEMENTED); + + if (ram_size_needed > actual_ram) { - size_t ram_size_needed = config->getUInt64("preconditions.ram_size"); - size_t actual_ram = system_information->totalram / 1024 / 1024; - if (ram_size_needed > actual_ram) - { - std::cerr << "Not enough RAM" << std::endl; - delete system_information; - return false; - } + std::cerr << "Not enough RAM: need = " << ram_size_needed << ", present = " << actual_ram << std::endl; + return false; } -#else - throw DB::Exception("ram_size precondition not available on this platform", ErrorCodes::NOT_IMPLEMENTED); -#endif } if (precondition == "table_exists") @@ -706,11 +694,15 @@ private: { Connection::Packet packet = connection.receivePacket(); - if (packet.type == Protocol::Server::Data) { + if (packet.type == Protocol::Server::Data) + { for (const ColumnWithTypeAndName & column : packet.block.getColumns()) { - if (column.name == "result" && column.column->getDataAt(0).data != nullptr) { + if (column.name == "result" && column.column->size() > 0) + { exist = column.column->get64(0); + if (exist) + break; } } } @@ -719,8 +711,9 @@ private: break; } - if (exist == 0) { - std::cerr << "Table " + table_to_check + " doesn't exist" << std::endl; + if (!exist) + { + std::cerr << "Table " << table_to_check << " doesn't exist" << std::endl; return false; } } diff --git a/libs/libcommon/CMakeLists.txt b/libs/libcommon/CMakeLists.txt index a86e6f2552..ed90c67e04 100644 --- a/libs/libcommon/CMakeLists.txt +++ b/libs/libcommon/CMakeLists.txt @@ -26,6 +26,7 @@ add_library (common src/DateLUTImpl.cpp src/exp10.cpp src/JSON.cpp + src/getMemoryAmount.cpp include/common/ApplicationServerExt.h include/common/Types.h @@ -41,6 +42,7 @@ add_library (common include/common/singleton.h include/common/strong_typedef.h include/common/JSON.h + include/common/getMemoryAmount.h include/ext/bit_cast.h include/ext/collection_cast.h diff --git a/libs/libcommon/include/common/getMemoryAmount.h b/libs/libcommon/include/common/getMemoryAmount.h new file mode 100644 index 0000000000..5139c39deb --- /dev/null +++ b/libs/libcommon/include/common/getMemoryAmount.h @@ -0,0 +1,9 @@ +#pragma once + +#include + +/** +* Returns the size of physical memory (RAM) in bytes. +* Returns 0 on unsupported platform +*/ +uint64_t getMemoryAmount(); diff --git a/libs/libcommon/include/common/iostream_debug_helpers.h b/libs/libcommon/include/common/iostream_debug_helpers.h index 39e16672ef..fdeb92ad5d 100644 --- a/libs/libcommon/include/common/iostream_debug_helpers.h +++ b/libs/libcommon/include/common/iostream_debug_helpers.h @@ -160,4 +160,13 @@ std::ostream & operator<<(std::ostream & stream, const std::experimental::option } +#include + +std::ostream & operator<<(std::ostream & stream, const std::exception & what) +{ + stream << "exception{" << what.what() << "}"; + return stream; +} + + // TODO: add more types diff --git a/libs/libcommon/src/getMemoryAmount.cpp b/libs/libcommon/src/getMemoryAmount.cpp new file mode 100644 index 0000000000..36551db62e --- /dev/null +++ b/libs/libcommon/src/getMemoryAmount.cpp @@ -0,0 +1,97 @@ +#include "common/getMemoryAmount.h" + +// http://nadeausoftware.com/articles/2012/09/c_c_tip_how_get_physical_memory_size_system + +/* + * Author: David Robert Nadeau + * Site: http://NadeauSoftware.com/ + * License: Creative Commons Attribution 3.0 Unported License + * http://creativecommons.org/licenses/by/3.0/deed.en_US + */ + +#if defined(WIN32) || defined(_WIN32) +#include +#else +#include +#include +#include +#if defined(BSD) +#include +#endif +#endif + + +/** + * Returns the size of physical memory (RAM) in bytes. + * Returns 0 on unsupported platform + */ +uint64_t getMemoryAmount() +{ +#if defined(_WIN32) && (defined(__CYGWIN__) || defined(__CYGWIN32__)) + /* Cygwin under Windows. ------------------------------------ */ + /* New 64-bit MEMORYSTATUSEX isn't available. Use old 32.bit */ + MEMORYSTATUS status; + status.dwLength = sizeof(status); + GlobalMemoryStatus( &status ); + return status.dwTotalPhys; + +#elif defined(WIN32) || defined(_WIN32) + /* Windows. ------------------------------------------------- */ + /* Use new 64-bit MEMORYSTATUSEX, not old 32-bit MEMORYSTATUS */ + MEMORYSTATUSEX status; + status.dwLength = sizeof(status); + GlobalMemoryStatusEx( &status ); + return status.ullTotalPhys; + +#else + /* UNIX variants. ------------------------------------------- */ + /* Prefer sysctl() over sysconf() except sysctl() HW_REALMEM and HW_PHYSMEM */ + +#if defined(CTL_HW) && (defined(HW_MEMSIZE) || defined(HW_PHYSMEM64)) + int mib[2]; + mib[0] = CTL_HW; +#if defined(HW_MEMSIZE) + mib[1] = HW_MEMSIZE; /* OSX. --------------------- */ +#elif defined(HW_PHYSMEM64) + mib[1] = HW_PHYSMEM64; /* NetBSD, OpenBSD. --------- */ +#endif + uint64_t size = 0; /* 64-bit */ + size_t len = sizeof(size); + if ( sysctl( mib, 2, &size, &len, NULL, 0 ) == 0 ) { + return size; + } + return 0; /* Failed? */ + +#elif defined(_SC_AIX_REALMEM) + /* AIX. ----------------------------------------------------- */ + return sysconf( _SC_AIX_REALMEM ) * 1024; + +#elif defined(_SC_PHYS_PAGES) && defined(_SC_PAGESIZE) + /* FreeBSD, Linux, OpenBSD, and Solaris. -------------------- */ + return (uint64_t)sysconf( _SC_PHYS_PAGES ) + * (uint64_t)sysconf( _SC_PAGESIZE ); + +#elif defined(_SC_PHYS_PAGES) && defined(_SC_PAGE_SIZE) + /* Legacy. -------------------------------------------------- */ + return (uint64_t)sysconf( _SC_PHYS_PAGES ) + * (uint64_t)sysconf( _SC_PAGE_SIZE ); + +#elif defined(CTL_HW) && (defined(HW_PHYSMEM) || defined(HW_REALMEM)) + /* DragonFly BSD, FreeBSD, NetBSD, OpenBSD, and OSX. -------- */ + int mib[2]; + mib[0] = CTL_HW; +#if defined(HW_REALMEM) + mib[1] = HW_REALMEM; /* FreeBSD. ----------------- */ +#elif defined(HW_PYSMEM) + mib[1] = HW_PHYSMEM; /* Others. ------------------ */ +#endif + unsigned int size = 0; /* 32-bit */ + size_t len = sizeof( size ); + if ( sysctl( mib, 2, &size, &len, NULL, 0 ) == 0 ) { + return size; + } + return 0; /* Failed? */ +#endif /* sysctl and sysconf variants */ + +#endif +} -- GitLab