CPU additionnal stats monitoring: Context switch, Interrupts... (issue #810)

NEWS

@@ -7,6 +7,7 @@ Version 2.7
 
 Enhancements and new features:
 
+    * CPU additionnal stats monitoring: Context switch, Interrupts... (issue #810)
     * [Folders] Differentiate permission issue and non-existence of a directory (issue #828)
     * [Web UI] add cpu name in quicklook plugin (issue #825)
 

conf/glances.conf

@@ -28,6 +28,11 @@ steal_careful=50
 steal_warning=70
 steal_critical=90
 #steal_log=True
+# Context switch limit per core / second
+# For example, if you have 2 Core, critical limit will be 28000/sec
+ctx_switches_careful=10000
+ctx_switches_warning=12000
+ctx_switches_critical=14000
 
 [percpu]
 # Define CPU thresholds in %

docs/aoa/cpu.rst

@@ -3,8 +3,8 @@
 CPU
 ===
 
-The CPU stats are shown as a percentage and for the configured refresh
-time. The total CPU usage is displayed on the first line.
+The CPU stats are shown as a percentage or value and for the configured
+refresh time. The total CPU usage is displayed on the first line.
 
 .. image:: ../_static/cpu.png
 
@@ -13,6 +13,20 @@ displayed.
 
 .. image:: ../_static/cpu-wide.png
 
+CPU stats description:
+
+* user: percent time spent in user space
+* system: percent time spent in kernel space
+* idle: percent of CPU used by any program
+* nice: percent time occupied by user level processes with a positive nice value
+* irq: percent time spent servicing/handling hardware/software interrupts
+* iowait: percent time spent in wait (on disk)
+* steal: percent time in involuntary wait by virtual cpu while hypervisor is servicing another processor/virtual machine
+* ctx_sw: number of context switches (voluntary + involuntary) per second
+* inter: number of interrupts per second
+* sw_inter: number of software interrupts per second. Always set to 0 on Windows and SunOS.
+* syscal: number of system calls per second. Do not displayed on Linux (always 0).
+
 To switch to per-CPU stats, just hit the ``1`` key:
 
 .. image:: ../_static/per-cpu.png

docs/man/glances.1

 .\" Man page generated from reStructuredText.
 .
-.TH "GLANCES" "1" "March 28, 2016" "2.7_BETA" "Glances"
+.TH "GLANCES" "1" "March 31, 2016" "2.7_BETA" "Glances"
 .SH NAME
 glances \- An eye on your system
 .

glances/plugins/glances_cpu.py

@@ -19,8 +19,11 @@
 
 """CPU plugin."""
 
+from glances.timer import getTimeSinceLastUpdate
 from glances.compat import iterkeys
 from glances.cpu_percent import cpu_percent
+from glances.globals import LINUX
+from glances.plugins.glances_core import Plugin as CorePlugin
 from glances.plugins.glances_plugin import GlancesPlugin
 
 import psutil
@@ -65,6 +68,12 @@ class Plugin(GlancesPlugin):
         # Init stats
         self.reset()
 
+        # Call CorePlugin in order to display the core number
+        try:
+            self.nb_log_core = CorePlugin(args=self.args).update()["log"]
+        except Exception:
+            self.nb_log_core = 1
+
     def reset(self):
         """Reset/init the stats."""
         self.stats = {}
@@ -75,9 +84,23 @@ class Plugin(GlancesPlugin):
         # Reset stats
         self.reset()
 
-        # Grab CPU stats using psutil's cpu_percent and cpu_times_percent
-        # methods
+        # Grab stats into self.stats
         if self.input_method == 'local':
+            self.update_local()
+        elif self.input_method == 'snmp':
+            self.update_snmp()
+
+        # Update the history list
+        self.update_stats_history()
+
+        # Update the view
+        self.update_views()
+
+        return self.stats
+
+    def update_local(self):
+        """Update CPU stats using PSUtil."""
+        # Grab CPU stats using psutil's cpu_percent and cpu_times_percent
         # Get all possible values for CPU stats: user, system, idle,
         # nice (UNIX), iowait (Linux), irq (Linux, FreeBSD), steal (Linux 2.6.11+)
         # The following stats are returned by the API but not displayed in the UI:
@@ -88,7 +111,41 @@ class Plugin(GlancesPlugin):
                      'irq', 'softirq', 'steal', 'guest', 'guest_nice']:
             if hasattr(cpu_times_percent, stat):
                 self.stats[stat] = getattr(cpu_times_percent, stat)
-        elif self.input_method == 'snmp':
+
+        # Additionnal CPU stats (number of events / not as a %)
+        # ctx_switches: number of context switches (voluntary + involuntary) per second
+        # interrupts: number of interrupts per second
+        # soft_interrupts: number of software interrupts per second. Always set to 0 on Windows and SunOS.
+        # syscalls: number of system calls since boot. Always set to 0 on Linux.
+        try:
+            cpu_stats = psutil.cpu_stats()
+        except AttributeError:
+            # cpu_stats only available with PSUtil 4.1 or +
+            pass
+        else:
+            # By storing time data we enable Rx/s and Tx/s calculations in the
+            # XML/RPC API, which would otherwise be overly difficult work
+            # for users of the API
+            time_since_update = getTimeSinceLastUpdate('cpu')
+
+            # Previous CPU stats are stored in the cpu_stats_old variable
+            if not hasattr(self, 'cpu_stats_old'):
+                # First call, we init the cpu_stats_old var
+                self.cpu_stats_old = cpu_stats
+            else:
+                for stat in cpu_stats._fields:
+                    self.stats[stat] = getattr(cpu_stats, stat) - getattr(self.cpu_stats_old, stat)
+
+                self.stats['time_since_update'] = time_since_update
+
+                # Core number is needed to compute the CTX switch limit
+                self.stats['cpucore'] = self.nb_log_core
+
+                # Save stats to compute next step
+                self.cpu_stats_old = cpu_stats
+
+    def update_snmp(self):
+        """Update CPU stats using SNMP."""
         # Update stats using SNMP
         if self.short_system_name in ('windows', 'esxi'):
             # Windows or VMWare ESXi
@@ -131,14 +188,6 @@ class Plugin(GlancesPlugin):
                 self.stats[key] = float(self.stats[key])
             self.stats['total'] = 100 - self.stats['idle']
 
-        # Update the history list
-        self.update_stats_history()
-
-        # Update the view
-        self.update_views()
-
-        return self.stats
-
     def update_views(self):
         """Update stats views."""
         # Call the father's method
@@ -153,8 +202,12 @@ class Plugin(GlancesPlugin):
         for key in ['steal', 'total']:
             if key in self.stats:
                 self.views[key]['decoration'] = self.get_alert(self.stats[key], header=key)
+        # Alert only but depend on Core number
+        for key in ['ctx_switches']:
+            if key in self.stats:
+                self.views[key]['decoration'] = self.get_alert(self.stats[key], maximum=100 * self.stats['cpucore'], header=key)
         # Optional
-        for key in ['nice', 'irq', 'iowait', 'steal']:
+        for key in ['nice', 'irq', 'iowait', 'steal', 'ctx_switches', 'interrupts', 'soft_interrupts', 'syscalls']:
             if key in self.stats:
                 self.views[key]['optional'] = True
 
@@ -171,6 +224,7 @@ class Plugin(GlancesPlugin):
         # If user stat is not here, display only idle / total CPU usage (for
         # exemple on Windows OS)
         idle_tag = 'user' not in self.stats
+
         # Header
         msg = '{0:8}'.format('CPU')
         ret.append(self.curse_add_line(msg, "TITLE"))
@@ -187,6 +241,15 @@ class Plugin(GlancesPlugin):
             ret.append(self.curse_add_line(msg, optional=self.get_views(key='nice', option='optional')))
             msg = '{0:>5}%'.format(self.stats['nice'])
             ret.append(self.curse_add_line(msg, optional=self.get_views(key='nice', option='optional')))
+        # ctx_switches
+        if 'ctx_switches' in self.stats:
+            msg = '  {0:8}'.format('ctx_sw:')
+            ret.append(self.curse_add_line(msg, optional=self.get_views(key='ctx_switches', option='optional')))
+            msg = '{0:>5}'.format(int(self.stats['ctx_switches'] // self.stats['time_since_update']))
+            ret.append(self.curse_add_line(
+                msg, self.get_views(key='ctx_switches', option='decoration'),
+                optional=self.get_views(key='ctx_switches', option='optional')))
+
         # New line
         ret.append(self.curse_new_line())
         # User CPU
@@ -207,6 +270,13 @@ class Plugin(GlancesPlugin):
             ret.append(self.curse_add_line(msg, optional=self.get_views(key='irq', option='optional')))
             msg = '{0:>5}%'.format(self.stats['irq'])
             ret.append(self.curse_add_line(msg, optional=self.get_views(key='irq', option='optional')))
+        # interrupts
+        if 'interrupts' in self.stats:
+            msg = '  {0:8}'.format('inter:')
+            ret.append(self.curse_add_line(msg, optional=self.get_views(key='interrupts', option='optional')))
+            msg = '{0:>5}'.format(int(self.stats['interrupts'] // self.stats['time_since_update']))
+            ret.append(self.curse_add_line(msg, optional=self.get_views(key='interrupts', option='optional')))
+
         # New line
         ret.append(self.curse_new_line())
         # System CPU
@@ -229,6 +299,13 @@ class Plugin(GlancesPlugin):
             ret.append(self.curse_add_line(
                 msg, self.get_views(key='iowait', option='decoration'),
                 optional=self.get_views(key='iowait', option='optional')))
+        # soft_interrupts
+        if 'soft_interrupts' in self.stats:
+            msg = '  {0:8}'.format('sw_int:')
+            ret.append(self.curse_add_line(msg, optional=self.get_views(key='soft_interrupts', option='optional')))
+            msg = '{0:>5}'.format(int(self.stats['soft_interrupts'] // self.stats['time_since_update']))
+            ret.append(self.curse_add_line(msg, optional=self.get_views(key='soft_interrupts', option='optional')))
+
         # New line
         ret.append(self.curse_new_line())
         # Idle CPU
@@ -245,6 +322,13 @@ class Plugin(GlancesPlugin):
             ret.append(self.curse_add_line(
                 msg, self.get_views(key='steal', option='decoration'),
                 optional=self.get_views(key='steal', option='optional')))
+        # syscalls
+        # syscalls: number of system calls since boot. Always set to 0 on Linux. (do not display)
+        if 'syscalls' in self.stats and not LINUX:
+            msg = '  {0:8}'.format('syscal:')
+            ret.append(self.curse_add_line(msg, optional=self.get_views(key='syscalls', option='optional')))
+            msg = '{0:>5}'.format(int(self.stats['syscalls'] // self.stats['time_since_update']))
+            ret.append(self.curse_add_line(msg, optional=self.get_views(key='syscalls', option='optional')))
 
         # Return the message with decoration
         return ret

glances/plugins/glances_diskio.py

@@ -81,7 +81,7 @@ class Plugin(GlancesPlugin):
 
             # Previous disk IO stats are stored in the diskio_old variable
             if not hasattr(self, 'diskio_old'):
-                # First call, we init the network_old var
+                # First call, we init the diskio_old var
                 try:
                     self.diskio_old = diskiocounters
                 except (IOError, UnboundLocalError):

glances/plugins/glances_load.py

@@ -62,7 +62,7 @@ class Plugin(GlancesPlugin):
         try:
             self.nb_log_core = CorePlugin(args=self.args).update()["log"]
         except Exception:
-            self.nb_log_core = 0
+            self.nb_log_core = 1
 
     def reset(self):
         """Reset/init the stats."""

glances/plugins/glances_plugin.py

@@ -403,6 +403,8 @@ class GlancesPlugin(object):
         except KeyError:
             return 'DEFAULT'
 
+        logger.debug("{0} => ret = {1}".format(stat_name, ret))
+
         # Manage log
         log_str = ""
         if self.__get_limit_log(stat_name=stat_name, default_action=log):
@@ -455,6 +457,8 @@ class GlancesPlugin(object):
             # Exemple: network_careful
             limit = self._limits[self.plugin_name + '_' + criticity]
 
+        # logger.debug("{0} {1} value is {2}".format(stat_name, criticity, limit))
+
         # Return the limit
         return limit
 

glances/plugins/glances_quicklook.py

@@ -113,12 +113,13 @@ class Plugin(GlancesPlugin):
         bar = Bar(max_width)
 
         # Build the string message
-        if 'cpu_name' in self.stats:
-            msg = '{0} - {1:.2f}/{2:.2f}GHz'.format(self.stats['cpu_name'],
-                                                    self._hz_to_ghz(self.stats['cpu_hz_current']),
+        if 'cpu_name' in self.stats and 'cpu_hz_current' in self.stats and 'cpu_hz' in self.stats:
+            msg_name = '{0} - '.format(self.stats['cpu_name'])
+            msg_freq = '{0:.2f}/{1:.2f}GHz'.format(self._hz_to_ghz(self.stats['cpu_hz_current']),
                                                    self._hz_to_ghz(self.stats['cpu_hz']))
-            if len(msg) - 6 <= max_width:
-                ret.append(self.curse_add_line(msg))
+            if len(msg_name + msg_freq) - 6 <= max_width:
+                ret.append(self.curse_add_line(msg_name))
+            ret.append(self.curse_add_line(msg_freq))
             ret.append(self.curse_new_line())
         for key in ['cpu', 'mem', 'swap']:
             if key == 'cpu' and args.percpu: