Merge pull request #3016 from yandex/metrics-documentation

Added documentation for CurrentMetrics #3010

dbms/programs/server/MetricsTransmitter.cpp

@@ -85,7 +85,7 @@ void MetricsTransmitter::transmit(std::vector<ProfileEvents::Count> & prev_count
             const auto counter_increment = counter - prev_counters[i];
             prev_counters[i] = counter;
 
-            std::string key{ProfileEvents::getDescription(static_cast<ProfileEvents::Event>(i))};
+            std::string key{ProfileEvents::getName(static_cast<ProfileEvents::Event>(i))};
             key_vals.emplace_back(profile_events_path_prefix + key, counter_increment);
         }
     }
@@ -96,7 +96,7 @@ void MetricsTransmitter::transmit(std::vector<ProfileEvents::Count> & prev_count
         {
             const auto value = CurrentMetrics::values[i].load(std::memory_order_relaxed);
 
-            std::string key{CurrentMetrics::getDescription(static_cast<CurrentMetrics::Metric>(i))};
+            std::string key{CurrentMetrics::getName(static_cast<CurrentMetrics::Metric>(i))};
             key_vals.emplace_back(current_metrics_path_prefix + key, value);
         }
     }

dbms/src/Common/CurrentMetrics.cpp

@@ -3,68 +3,80 @@
 
 /// Available metrics. Add something here as you wish.
 #define APPLY_FOR_METRICS(M) \
-    M(Query) \
-    M(Merge) \
-    M(PartMutation) \
-    M(ReplicatedFetch) \
-    M(ReplicatedSend) \
-    M(ReplicatedChecks) \
-    M(BackgroundPoolTask) \
-    M(BackgroundSchedulePoolTask) \
-    M(DiskSpaceReservedForMerge) \
-    M(DistributedSend) \
-    M(QueryPreempted) \
-    M(TCPConnection) \
-    M(HTTPConnection) \
-    M(InterserverConnection) \
-    M(OpenFileForRead) \
-    M(OpenFileForWrite) \
-    M(Read) \
-    M(Write) \
-    M(SendExternalTables) \
-    M(QueryThread) \
-    M(ReadonlyReplica) \
-    M(LeaderReplica) \
-    M(MemoryTracking) \
-    M(MemoryTrackingInBackgroundProcessingPool) \
-    M(MemoryTrackingInBackgroundSchedulePool) \
-    M(MemoryTrackingForMerges) \
-    M(LeaderElection) \
-    M(EphemeralNode) \
-    M(ZooKeeperSession) \
-    M(ZooKeeperWatch) \
-    M(ZooKeeperRequest) \
-    M(DelayedInserts) \
-    M(ContextLockWait) \
-    M(StorageBufferRows) \
-    M(StorageBufferBytes) \
-    M(DictCacheRequests) \
-    M(Revision) \
-    M(RWLockWaitingReaders) \
-    M(RWLockWaitingWriters) \
-    M(RWLockActiveReaders) \
-    M(RWLockActiveWriters)
+    M(Query, "Number of executing queries") \
+    M(Merge, "Number of executing background merges") \
+    M(PartMutation, "Number of mutations (ALTER DELETE/UPDATE)") \
+    M(ReplicatedFetch, "Number of data parts fetching from replica") \
+    M(ReplicatedSend, "Number of data parts sending to replicas") \
+    M(ReplicatedChecks, "Number of data parts checking for consistency") \
+    M(BackgroundPoolTask, "Number of active tasks in BackgroundProcessingPool (merges, mutations, fetches or replication queue bookkeeping)") \
+    M(BackgroundSchedulePoolTask, "Number of active tasks in BackgroundSchedulePool. This pool is used for periodic tasks of ReplicatedMergeTree like cleaning old data parts, altering data parts, replica re-initialization, etc.") \
+    M(DiskSpaceReservedForMerge, "Disk space reserved for currently running background merges. It is slightly more than total size of currently merging parts.") \
+    M(DistributedSend, "Number of connections sending data, that was INSERTed to Distributed tables, to remote servers. Both synchronous and asynchronous mode.") \
+    M(QueryPreempted, "Number of queries that are stopped and waiting due to 'priority' setting.") \
+    M(TCPConnection, "Number of connections to TCP server (clients with native interface)") \
+    M(HTTPConnection, "Number of connections to HTTP server") \
+    M(InterserverConnection, "Number of connections from other replicas to fetch parts") \
+    M(OpenFileForRead, "Number of files open for reading") \
+    M(OpenFileForWrite, "Number of files open for writing") \
+    M(Read, "Number of read (read, pread, io_getevents, etc.) syscalls in fly") \
+    M(Write, "Number of write (write, pwrite, io_getevents, etc.) syscalls in fly") \
+    M(SendExternalTables, "Number of connections that are sending data for external tables to remote servers. External tables are used to implement GLOBAL IN and GLOBAL JOIN operators with distributed subqueries.") \
+    M(QueryThread, "Number of query processing threads") \
+    M(ReadonlyReplica, "Number of Replicated tables that are currently in readonly state due to re-initialization after ZooKeeper session loss or due to startup without ZooKeeper configured.") \
+    M(LeaderReplica, "Number of Replicated tables that are leaders. Leader replica is responsible for assigning merges, cleaning old blocks for deduplications and a few more bookkeeping tasks. There may be no more than one leader across all replicas at one moment of time. If there is no leader it will be elected soon or it indicate an issue.") \
+    M(MemoryTracking, "Total amount of memory (bytes) allocated in currently executing queries. Note that some memory allocations may not be accounted.") \
+    M(MemoryTrackingInBackgroundProcessingPool, "Total amount of memory (bytes) allocated in background processing pool (that is dedicated for backround merges, mutations and fetches). Note that this value may include a drift when the memory was allocated in a context of background processing pool and freed in other context or vice-versa. This happens naturally due to caches for tables indexes and doesn't indicate memory leaks.") \
+    M(MemoryTrackingInBackgroundSchedulePool, "Total amount of memory (bytes) allocated in background schedule pool (that is dedicated for bookkeeping tasks of Replicated tables).") \
+    M(MemoryTrackingForMerges, "Total amount of memory (bytes) allocated for background merges. Included in MemoryTrackingInBackgroundProcessingPool. Note that this value may include a drift when the memory was allocated in a context of background processing pool and freed in other context or vice-versa. This happens naturally due to caches for tables indexes and doesn't indicate memory leaks.") \
+    M(LeaderElection, "Number of Replicas participating in leader election. Equals to total number of replicas in usual cases.") \
+    M(EphemeralNode, "Number of ephemeral nodes hold in ZooKeeper.") \
+    M(ZooKeeperSession, "Number of sessions (connections) to ZooKeeper. Should be no more than one, because using more than one connection to ZooKeeper may lead to bugs due to lack of linearizability (stale reads) that ZooKeeper consistency model allows.") \
+    M(ZooKeeperWatch, "Number of watches (event subscriptions) in ZooKeeper.") \
+    M(ZooKeeperRequest, "Number of requests to ZooKeeper in fly.") \
+    M(DelayedInserts, "Number of INSERT queries that are throttled due to high number of active data parts for partition in a MergeTree table.") \
+    M(ContextLockWait, "Number of threads waiting for lock in Context. This is global lock.") \
+    M(StorageBufferRows, "Number of rows in buffers of Buffer tables") \
+    M(StorageBufferBytes, "Number of bytes in buffers of Buffer tables") \
+    M(DictCacheRequests, "Number of requests in fly to data sources of dictionaries of cache type.") \
+    M(Revision, "Revision of the server. It is a number incremented for every release or release candidate.") \
+    M(RWLockWaitingReaders, "Number of threads waiting for read on a table RWLock.") \
+    M(RWLockWaitingWriters, "Number of threads waiting for write on a table RWLock.") \
+    M(RWLockActiveReaders, "Number of threads holding read lock in a table RWLock.") \
+    M(RWLockActiveWriters, "Number of threads holding write lock in a table RWLock.") \
 
 
 namespace CurrentMetrics
 {
-    #define M(NAME) extern const Metric NAME = __COUNTER__;
+    #define M(NAME, DOCUMENTATION) extern const Metric NAME = __COUNTER__;
         APPLY_FOR_METRICS(M)
     #undef M
     constexpr Metric END = __COUNTER__;
 
     std::atomic<Value> values[END] {};    /// Global variable, initialized by zeros.
 
-    const char * getDescription(Metric event)
+    const char * getName(Metric event)
     {
-        static const char * descriptions[] =
+        static const char * strings[] =
         {
-        #define M(NAME) #NAME,
+        #define M(NAME, DOCUMENTATION) #NAME,
             APPLY_FOR_METRICS(M)
         #undef M
         };
 
-        return descriptions[event];
+        return strings[event];
+    }
+
+    const char * getDocumentation(Metric event)
+    {
+        static const char * strings[] =
+        {
+        #define M(NAME, DOCUMENTATION) DOCUMENTATION,
+            APPLY_FOR_METRICS(M)
+        #undef M
+        };
+
+        return strings[event];
     }
 
     Metric end() { return END; }

dbms/src/Common/CurrentMetrics.h

@@ -24,8 +24,10 @@ namespace CurrentMetrics
     using Metric = size_t;
     using Value = DB::Int64;
 
+    /// Get name of metric by identifier. Returns statically allocated string.
+    const char * getName(Metric event);
     /// Get text description of metric by identifier. Returns statically allocated string.
-    const char * getDescription(Metric event);
+    const char * getDocumentation(Metric event);
 
     /// Metric identifier -> current value of metric.
     extern std::atomic<Value> values[];

dbms/src/Common/ProfileEvents.cpp

@@ -218,16 +218,16 @@ Counters Counters::getPartiallyAtomicSnapshot() const
     return res;
 }
 
-const char * getDescription(Event event)
+const char * getName(Event event)
 {
-    static const char * descriptions[] =
+    static const char * strings[] =
     {
     #define M(NAME) #NAME,
         APPLY_FOR_EVENTS(M)
     #undef M
     };
 
-    return descriptions[event];
+    return strings[event];
 }
 
 

dbms/src/Common/ProfileEvents.h

@@ -86,8 +86,8 @@ namespace ProfileEvents
     /// Increment a counter for event. Thread-safe.
     void increment(Event event, Count amount = 1);
 
-    /// Get text description of event by identifier. Returns statically allocated string.
-    const char * getDescription(Event event);
+    /// Get name of event by identifier. Returns statically allocated string.
+    const char * getName(Event event);
 
     /// Get index just after last event identifier.
     Event end();

dbms/src/Interpreters/ProfileEventsExt.cpp

@@ -30,7 +30,7 @@ void dumpToArrayColumns(const Counters & counters, DB::IColumn * column_names_,
 
         if (column_names)
         {
-            const char * desc = ProfileEvents::getDescription(event);
+            const char * desc = ProfileEvents::getName(event);
             column_names->getData().insertData(desc, strlen(desc));
         }
 

dbms/src/Storages/System/StorageSystemEvents.cpp

@@ -22,7 +22,7 @@ void StorageSystemEvents::fillData(MutableColumns & res_columns, const Context &
 
         if (0 != value)
         {
-            res_columns[0]->insert(String(ProfileEvents::getDescription(ProfileEvents::Event(i))));
+            res_columns[0]->insert(String(ProfileEvents::getName(ProfileEvents::Event(i))));
             res_columns[1]->insert(value);
         }
     }

dbms/src/Storages/System/StorageSystemMetrics.cpp

@@ -13,6 +13,7 @@ NamesAndTypesList StorageSystemMetrics::getNamesAndTypes()
     return {
         {"metric", std::make_shared<DataTypeString>()},
         {"value", std::make_shared<DataTypeInt64>()},
+        {"description", std::make_shared<DataTypeString>()},
     };
 }
 
@@ -22,8 +23,9 @@ void StorageSystemMetrics::fillData(MutableColumns & res_columns, const Context
     {
         Int64 value = CurrentMetrics::values[i].load(std::memory_order_relaxed);
 
-        res_columns[0]->insert(String(CurrentMetrics::getDescription(CurrentMetrics::Metric(i))));
+        res_columns[0]->insert(String(CurrentMetrics::getName(CurrentMetrics::Metric(i))));
         res_columns[1]->insert(value);
+        res_columns[2]->insert(String(CurrentMetrics::getDocumentation(CurrentMetrics::Metric(i))));
     }
 }