Generate specific tag from stats' key for InfluxDB and InfluxDB2 export #1871

conf/glances.conf

@@ -449,12 +449,11 @@ db=glances
 #     => foo.cpu
 #     => foo.mem
 # You can also use dynamic values
-#prefix=`hostname`
-prefix=localhost
-# Tags will be added for all measurements
-#tags=foo:bar,spam:eggs
-# You can also use dynamic values
-#tags=system:`uname -s`
+#prefix=foo
+# Followings tags will be added for all measurements
+# You can also use dynamic values.
+# Note: hostname is always added as a tag
+#tags=foo:bar,spam:eggs,domain:`domainname`
 
 [influxdb2]
 # Configuration for the --export influxdb2 option
@@ -470,12 +469,11 @@ token=EjFUTWe8U-MIseEAkaVIgVnej_TrnbdvEcRkaB1imstW7gapSqy6_6-8XD-yd51V0zUUpDy-kA
 #     => foo.cpu
 #     => foo.mem
 # You can also use dynamic values
-#prefix=`hostname`
-prefix=localhost
-# Tags will be added for all measurements
-#tags=foo:bar,spam:eggs
-# You can also use dynamic values
-#tags=system:`uname -s`
+#prefix=foo
+# Followings tags will be added for all measurements
+# You can also use dynamic values.
+# Note: hostname is always added as a tag
+#tags=foo:bar,spam:eggs,domain:`domainname`
 
 [cassandra]
 # Configuration for the --export cassandra option

docs/gw/influxdb.rst

@@ -5,9 +5,29 @@ InfluxDB
 
 You can export statistics to an ``InfluxDB`` server (time series server).
 
-In Glances version 3.1.8 and higher, the value of the key field will be
-converted as a tag. For example, in the Docker plugin, the key = name,
-so the container name will be tag and not a field.
+In Glances version 3.1.8 and higher, the way Glances exports stats to
+InfluxDB changes. The following fields will be added as tags:
+- key stats (for example *interface_name* for network, container *name* for docker...)
+- hostname (shortname)
+- tags
+
+Glances InfluxDB data model:
+
++---------------+-----------------------+-----------------------+
+| Measurement   | Fields                | Tags                  |
++===============+=======================+=======================+
+| cpu           | user                  | hostname              |
+|               | system                |                       |
+|               | iowait...             |                       |
++---------------+-----------------------+-----------------------+
+| network       | rx                    |                       |
+|               | tx                    |                       |
+|               | time_since_update...  | hostname              |
+|               |                       | interface_name        |
++---------------+-----------------------+-----------------------+
+| docker        | cpu_percent           | hostname              |
+|               | memory_usage...       | name                  |
++---------------+-----------------------+-----------------------+
 
 InfluxDB (up to version 1.7.x)
 ------------------------------
@@ -29,12 +49,11 @@ following:
     #     => foo.cpu
     #     => foo.mem
     # You can also use dynamic values
-    #prefix=`hostname`
-    prefix=localhost
-    # Tags will be added for all measurements
-    #tags=foo:bar,spam:eggs
-    # You can also use dynamic values
-    #tags=system:`uname -s`
+    #prefix=foo
+    # Followings tags will be added for all measurements
+    # You can also use dynamic values.
+    # Note: hostname is always added as a tag
+    #tags=foo:bar,spam:eggs,domain:`domainname`
 
 and run Glances with:
 
@@ -72,12 +91,11 @@ following:
     #     => foo.cpu
     #     => foo.mem
     # You can also use dynamic values
-    #prefix=`hostname`
-    prefix=localhost
-    # Tags will be added for all measurements
-    #tags=foo:bar,spam:eggs
-    # You can also use dynamic values
-    #tags=system:`uname -s`
+    #prefix=foo
+    # Followings tags will be added for all measurements
+    # You can also use dynamic values.
+    # Note: hostname is always added as a tag
+    #tags=foo:bar,spam:eggs,domain:`domainname`
 
 and run Glances with:
 

glances/exports/glances_influxdb.py

@@ -20,6 +20,7 @@
 """InfluxDB (up to InfluxDB 1.7.x) interface class."""
 
 import sys
+from platform import node
 
 from glances.logger import logger
 from glances.exports.glances_export import GlancesExport
@@ -56,6 +57,9 @@ class Export(GlancesExport):
         if not self.export_enable:
             sys.exit(2)
 
+        # The hostname is always add as a tag
+        self.hostname = node().split('.')[0]
+
         # Init the InfluxDB client
         self.client = self.init()
 
@@ -115,7 +119,7 @@ class Export(GlancesExport):
             else:
                 fields = data_dict
             # Transform to InfluxDB datamodel
-            # https://docs.influxdata.com/influxdb/v1.5/write_protocols/line_protocol_reference/
+            # https://docs.influxdata.com/influxdb/v1.8/write_protocols/line_protocol_reference/
             for k in fields:
                 #  Do not export empty (None) value
                 if fields[k] is None:
@@ -137,6 +141,8 @@ class Export(GlancesExport):
                 tags[fields['key']] = str(fields[fields['key']])
                 # Remove it from the field list (can not be a field and a tag)
                 fields.pop(fields['key'])
+            # Add the hostname as a tag
+            tags['hostname'] = self.hostname
             # Add the measurement to the list
             ret.append({'measurement': name,
                         'tags': tags,
@@ -153,7 +159,8 @@ class Export(GlancesExport):
             logger.debug("Cannot export empty {} stats to InfluxDB".format(name))
         else:
             try:
-                self.client.write_points(self._normalize(name, columns, points), time_precision="s")
+                self.client.write_points(self._normalize(name, columns, points),
+                                         time_precision="s")
             except Exception as e:
                 # Log level set to debug instead of error (see: issue #1561)
                 logger.debug("Cannot export {} stats to InfluxDB ({})".format(name, e))

glances/exports/glances_influxdb2.py

@@ -2,7 +2,7 @@
 #
 # This file is part of Glances.
 #
-# Copyright (C) 2020 Nicolargo <nicolas@nicolargo.com>
+# Copyright (C) 2021 Nicolargo <nicolas@nicolargo.com>
 #
 # Glances is free software; you can redistribute it and/or modify
 # it under the terms of the GNU Lesser General Public License as published by
@@ -67,7 +67,7 @@ class Export(GlancesExport):
         try:
             client = InfluxDBClient(url=url,
                                     enable_gzip=False,
-                                    org=self.org, 
+                                    org=self.org,
                                     token=self.token)
         except Exception as e:
             logger.critical("Cannot connect to InfluxDB server '%s' (%s)" % (url, e))
@@ -76,7 +76,6 @@ class Export(GlancesExport):
             logger.info("Connected to InfluxDB server version {} ({})".format(client.health().version,
                                                                               client.health().message))
 
-
         # Create the write client
         write_client = client.write_api(write_options=WriteOptions(batch_size=500,
                                                                    flush_interval=10000,
@@ -89,31 +88,55 @@ class Export(GlancesExport):
 
     def _normalize(self, name, columns, points):
         """Normalize data for the InfluxDB's data model."""
-
-        for i, _ in enumerate(points):
-            # Supported type:
-            # https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/
-            if points[i] is None:
-                # Ignore points with None value
-                del(points[i])
-                del(columns[i])
-                continue
-            try:
-                points[i] = float(points[i])
-            except (TypeError, ValueError):
-                pass
+        ret = []
+
+        # Build initial dict by crossing columns and point
+        data_dict = dict(zip(columns, points))
+
+        # issue1871 - Check if a key exist. If a key exist, the value of
+        # the key should be used as a tag to identify the measurement.
+        keys_list = [k.split('.')[0] for k in columns if k.endswith('.key')]
+        if len(keys_list) == 0:
+            keys_list = [None]
+
+        for measurement in keys_list:
+            # Manage field
+            if measurement is not None:
+                fields = {k.replace('{}.'.format(measurement), ''): data_dict[k]
+                          for k in data_dict
+                          if k.startswith('{}.'.format(measurement))}
             else:
-                continue
-            try:
-                points[i] = str(points[i])
-            except (TypeError, ValueError):
-                pass
-            else:
-                continue
-
-        return [{'measurement': name,
-                 'tags': self.parse_tags(self.tags),
-                 'fields': dict(zip(columns, points))}]
+                fields = data_dict
+            # Transform to InfluxDB datamodel
+            # https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/
+            for k in fields:
+                #  Do not export empty (None) value
+                if fields[k] is None:
+                    fields.pop(k)
+                # Convert numerical to float
+                try:
+                    fields[k] = float(fields[k])
+                except (TypeError, ValueError):
+                    # Convert others to string
+                    try:
+                        fields[k] = str(fields[k])
+                    except (TypeError, ValueError):
+                        pass
+            # Manage tags
+            tags = self.parse_tags(self.tags)
+            if 'key' in fields and fields['key'] in fields:
+                # Create a tag from the key
+                # Tag should be an string (see InfluxDB data model)
+                tags[fields['key']] = str(fields[fields['key']])
+                # Remove it from the field list (can not be a field and a tag)
+                fields.pop(fields['key'])
+            # Add the hostname as a tag
+            tags['hostname'] = self.hostname
+            # Add the measurement to the list
+            ret.append({'measurement': name,
+                        'tags': tags,
+                        'fields': fields})
+        return ret
 
     def export(self, name, columns, points):
         """Write the points to the InfluxDB server."""
@@ -127,7 +150,7 @@ class Export(GlancesExport):
             try:
                 self.client.write(self.bucket,
                                   self.org,
-                                  self._normalize(name, columns, points), 
+                                  self._normalize(name, columns, points),
                                   time_precision="s")
             except Exception as e:
                 # Log level set to debug instead of error (see: issue #1561)