Fixing PerformanceTest: table_exists condition, move getMemoryAmount function...

Fixing PerformanceTest: table_exists condition, move getMemoryAmount function to file. Add more debug helpers (#905)

* Fix missing includes

* ComplexKeyCacheDictionary: Move includes to .cpp

* Fixing PerformanceTest: table_exists condition, move getMemoryAmount function to file. Add more debug helpers

* Multiplatform memory size getter

* Requested changes

* Requested changes

* Requested changes

* fix

* Requested changes

dbms/src/Common/iostream_debug_helpers.cpp

@@ -62,7 +62,7 @@ std::ostream & operator<<(std::ostream & stream, const DB::Block & what)
 {
     stream << "Block("
            << "size = " << what.getColumns().size()
-           << ")";
+           << "){" << what.dumpStructure() << "}";
     return stream;
 }
 
@@ -79,3 +79,15 @@ std::ostream & operator<<(std::ostream & stream, const DB::IColumn & what)
            << ")";
     return stream;
 }
+
+std::ostream & operator<<(std::ostream & stream, const DB::Connection::Packet & what) {
+    stream << "Connection::Packet("
+        << "type = " << what.type;
+        // types description: Core/Protocol.h
+    if (what.exception)
+        stream << "exception = " << what.exception.get();
+    //TODO: profile_info
+    stream << ") {" << what.block << "}";
+    return stream;
+}
+

dbms/src/Common/iostream_debug_helpers.h

@@ -32,6 +32,9 @@ std::ostream & operator<<(std::ostream & stream, const DB::ColumnWithTypeAndName
 namespace DB { class IColumn; }
 std::ostream & operator<<(std::ostream & stream, const DB::IColumn & what);
 
+#include <Client/Connection.h>
+std::ostream & operator<<(std::ostream & stream, const DB::Connection::Packet & what);
+
 
 /// some operator<< should be declared before operator<<(... std::shared_ptr<>)
 #include <common/iostream_debug_helpers.h>

dbms/src/Dictionaries/ComplexKeyCacheDictionary.cpp

 #include <Dictionaries/ComplexKeyCacheDictionary.h>
 #include <Dictionaries/DictionaryBlockInputStream.h>
+#include <Common/Arena.h>
 #include <Common/BitHelpers.h>
 #include <Common/randomSeed.h>
 #include <Common/Stopwatch.h>
@@ -7,6 +8,8 @@
 #include <Common/ProfileEvents.h>
 #include <Common/CurrentMetrics.h>
 #include <ext/range.h>
+#include <ext/scope_guard.h>
+#include <ext/map.h>
 
 
 namespace ProfileEvents

dbms/src/Dictionaries/ComplexKeyCacheDictionary.h

@@ -3,15 +3,12 @@
 #include <Dictionaries/IDictionary.h>
 #include <Dictionaries/IDictionarySource.h>
 #include <Dictionaries/DictionaryStructure.h>
-#include <Common/Arena.h>
 #include <Common/ArenaWithFreeLists.h>
 #include <Common/SmallObjectPool.h>
 #include <Common/HashTable/HashMap.h>
 #include <Columns/ColumnString.h>
 #include <Core/StringRef.h>
-#include <ext/scope_guard.h>
 #include <ext/bit_cast.h>
-#include <ext/map.h>
 #include <Poco/RWLock.h>
 #include <atomic>
 #include <chrono>

dbms/src/Server/PerformanceTest.cpp

@@ -2,9 +2,6 @@
 #include <iostream>
 #include <limits>
 #include <regex>
-#if __has_include(<sys/sysinfo.h>)
-    #include <sys/sysinfo.h>
-#endif
 #include <unistd.h>
 
 #include <boost/program_options.hpp>
@@ -23,6 +20,7 @@
 #include <IO/ReadHelpers.h>
 #include <IO/WriteBufferFromFile.h>
 #include <Interpreters/Settings.h>
+#include <common/getMemoryAmount.h>
 
 
 #include <Poco/AutoPtr.h>
@@ -662,35 +660,25 @@ private:
         for (const String & precondition : preconditions)
         {
             if (precondition == "flush_disk_cache")
-                if (system("(>&2 echo 'Flushing disk cache...') && (sudo sh -c 'echo 3 > /proc/sys/vm/drop_caches') && (>&2 echo 'Flushed.')")) {
+                if (system("(>&2 echo 'Flushing disk cache...') && (sudo sh -c 'echo 3 > /proc/sys/vm/drop_caches') && (>&2 echo 'Flushed.')"))
+                {
                     std::cerr << "Failed to flush disk cache" << std::endl;
                     return false;
                 }
 
             if (precondition == "ram_size")
             {
-#if __has_include(<sys/sysinfo.h>)
-                struct sysinfo *system_information = new struct sysinfo();
-                if (sysinfo(system_information))
-                {
-                    std::cerr << "Failed to check system RAM size" << std::endl;
-                    delete system_information;
-                }
-                else
-                {
                 size_t ram_size_needed = config->getUInt64("preconditions.ram_size");
-                    size_t actual_ram = system_information->totalram / 1024 / 1024;
+                size_t actual_ram = getMemoryAmount();
+                if (!actual_ram)
+                    throw DB::Exception("ram_size precondition not available on this platform", ErrorCodes::NOT_IMPLEMENTED);
+
                 if (ram_size_needed > actual_ram)
                 {
-                        std::cerr << "Not enough RAM" << std::endl;
-                        delete system_information;
+                    std::cerr << "Not enough RAM: need = " << ram_size_needed << ", present = " << actual_ram << std::endl;
                     return false;
                 }
             }
-#else
-                throw DB::Exception("ram_size precondition not available on this platform", ErrorCodes::NOT_IMPLEMENTED);
-#endif
-            }
 
             if (precondition == "table_exists")
             {
@@ -706,11 +694,15 @@ private:
                 {
                     Connection::Packet packet = connection.receivePacket();
 
-                    if (packet.type == Protocol::Server::Data) {
+                    if (packet.type == Protocol::Server::Data)
+                    {
                         for (const ColumnWithTypeAndName & column : packet.block.getColumns())
                         {
-                            if (column.name == "result" && column.column->getDataAt(0).data != nullptr) {
+                            if (column.name == "result" && column.column->size() > 0)
+                            {
                                 exist = column.column->get64(0);
+                                if (exist)
+                                    break;
                             }
                         }
                     }
@@ -719,8 +711,9 @@ private:
                         break;
                 }
 
-                if (exist == 0) {
-                    std::cerr << "Table " + table_to_check + " doesn't exist" << std::endl;
+                if (!exist)
+                {
+                    std::cerr << "Table " << table_to_check << " doesn't exist" << std::endl;
                     return false;
                 }
             }

libs/libcommon/CMakeLists.txt

@@ -26,6 +26,7 @@ add_library (common
     src/DateLUTImpl.cpp
     src/exp10.cpp
     src/JSON.cpp
+    src/getMemoryAmount.cpp
 
     include/common/ApplicationServerExt.h
     include/common/Types.h
@@ -41,6 +42,7 @@ add_library (common
     include/common/singleton.h
     include/common/strong_typedef.h
     include/common/JSON.h
+    include/common/getMemoryAmount.h
 
     include/ext/bit_cast.h
     include/ext/collection_cast.h

libs/libcommon/include/common/getMemoryAmount.h

+#pragma once
+
+#include <cstdint>
+
+/**
+* Returns the size of physical memory (RAM) in bytes.
+* Returns 0 on unsupported platform
+*/
+uint64_t getMemoryAmount();

libs/libcommon/include/common/iostream_debug_helpers.h

@@ -160,4 +160,13 @@ std::ostream & operator<<(std::ostream & stream, const std::experimental::option
 }
 
 
+#include <exception>
+
+std::ostream & operator<<(std::ostream & stream, const std::exception & what)
+{
+    stream << "exception{" << what.what() << "}";
+    return stream;
+}
+
+
 // TODO: add more types

libs/libcommon/src/getMemoryAmount.cpp

+#include "common/getMemoryAmount.h"
+
+// http://nadeausoftware.com/articles/2012/09/c_c_tip_how_get_physical_memory_size_system
+
+/*
+ * Author:  David Robert Nadeau
+ * Site:    http://NadeauSoftware.com/
+ * License: Creative Commons Attribution 3.0 Unported License
+ *          http://creativecommons.org/licenses/by/3.0/deed.en_US
+ */
+
+#if defined(WIN32) || defined(_WIN32)
+#include <Windows.h>
+#else
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/param.h>
+#if defined(BSD)
+#include <sys/sysctl.h>
+#endif
+#endif
+
+
+/**
+ * Returns the size of physical memory (RAM) in bytes.
+ * Returns 0 on unsupported platform
+ */
+uint64_t getMemoryAmount()
+{
+#if defined(_WIN32) && (defined(__CYGWIN__) || defined(__CYGWIN32__))
+    /* Cygwin under Windows. ------------------------------------ */
+    /* New 64-bit MEMORYSTATUSEX isn't available.  Use old 32.bit */
+    MEMORYSTATUS status;
+    status.dwLength = sizeof(status);
+    GlobalMemoryStatus( &status );
+    return status.dwTotalPhys;
+
+#elif defined(WIN32) || defined(_WIN32)
+    /* Windows. ------------------------------------------------- */
+    /* Use new 64-bit MEMORYSTATUSEX, not old 32-bit MEMORYSTATUS */
+    MEMORYSTATUSEX status;
+    status.dwLength = sizeof(status);
+    GlobalMemoryStatusEx( &status );
+    return status.ullTotalPhys;
+
+#else
+    /* UNIX variants. ------------------------------------------- */
+    /* Prefer sysctl() over sysconf() except sysctl() HW_REALMEM and HW_PHYSMEM */
+
+#if defined(CTL_HW) && (defined(HW_MEMSIZE) || defined(HW_PHYSMEM64))
+    int mib[2];
+    mib[0] = CTL_HW;
+#if defined(HW_MEMSIZE)
+    mib[1] = HW_MEMSIZE; /* OSX. --------------------- */
+#elif defined(HW_PHYSMEM64)
+    mib[1] = HW_PHYSMEM64; /* NetBSD, OpenBSD. --------- */
+#endif
+    uint64_t size = 0; /* 64-bit */
+    size_t len = sizeof(size);
+    if ( sysctl( mib, 2, &size, &len, NULL, 0 ) == 0 ) {
+        return size;
+    }
+    return 0; /* Failed? */
+
+#elif defined(_SC_AIX_REALMEM)
+    /* AIX. ----------------------------------------------------- */
+    return sysconf( _SC_AIX_REALMEM ) * 1024;
+
+#elif defined(_SC_PHYS_PAGES) && defined(_SC_PAGESIZE)
+    /* FreeBSD, Linux, OpenBSD, and Solaris. -------------------- */
+    return (uint64_t)sysconf( _SC_PHYS_PAGES )
+           * (uint64_t)sysconf( _SC_PAGESIZE );
+
+#elif defined(_SC_PHYS_PAGES) && defined(_SC_PAGE_SIZE)
+    /* Legacy. -------------------------------------------------- */
+    return (uint64_t)sysconf( _SC_PHYS_PAGES )
+           * (uint64_t)sysconf( _SC_PAGE_SIZE );
+
+#elif defined(CTL_HW) && (defined(HW_PHYSMEM) || defined(HW_REALMEM))
+    /* DragonFly BSD, FreeBSD, NetBSD, OpenBSD, and OSX. -------- */
+    int mib[2];
+    mib[0] = CTL_HW;
+#if defined(HW_REALMEM)
+    mib[1] = HW_REALMEM; /* FreeBSD. ----------------- */
+#elif defined(HW_PYSMEM)
+    mib[1] = HW_PHYSMEM; /* Others. ------------------ */
+#endif
+    unsigned int size = 0; /* 32-bit */
+    size_t len = sizeof( size );
+    if ( sysctl( mib, 2, &size, &len, NULL, 0 ) == 0 ) {
+        return size;
+    }
+    return 0; /* Failed? */
+#endif /* sysctl and sysconf variants */
+
+#endif
+}