Ok but miss some extended stats (see line 348)

glances/amps_list.py

@@ -105,8 +105,7 @@ class AmpsList(object):
     def update(self):
         """Update the command result attributed."""
         # Search application monitored processes by a regular expression
-        processlist = glances_processes.getalllist()
-        logger.info(processlist)
+        processlist = glances_processes.getlist()
         # Iter upon the AMPs dict
         for k, v in iteritems(self.get()):
             if not v.enable():

glances/globals.py

@@ -25,6 +25,7 @@ import sys
 
 # Operating system flag
 # Note: Somes libs depends of OS
+# Note2: Included in PsUtil 4.0 or higher
 BSD = sys.platform.find('bsd') != -1
 LINUX = sys.platform.startswith('linux')
 MACOS = sys.platform.startswith('darwin')

glances/logs.py

@@ -109,7 +109,7 @@ class GlancesLogs(object):
         If 'item' is not a 'new one', update the existing item.
         If event < peak_time the the alert is not setoff.
         """
-        proc_list = proc_list or glances_processes.getalllist()
+        proc_list = proc_list or glances_processes.getlist()
 
         # Add or update the log
         item_index = self.__itemexist__(item_type)

glances/processes.py

@@ -20,7 +20,7 @@
 import operator
 import os
 
-from glances.compat import iteritems, itervalues, listitems
+from glances.compat import iteritems, itervalues, listitems, iterkeys
 from glances.globals import BSD, LINUX, MACOS, SUNOS, WINDOWS
 from glances.timer import Timer, getTimeSinceLastUpdate
 from glances.filter import GlancesFilter
@@ -29,16 +29,6 @@ from glances.logger import logger
 import psutil
 
 
-def is_kernel_thread(proc):
-    """Return True if proc is a kernel thread, False instead."""
-    try:
-        return os.getpgid(proc.pid) == 0
-    # Python >= 3.3 raises ProcessLookupError, which inherits OSError
-    except OSError:
-        # return False is process is dead
-        return False
-
-
 class GlancesProcesses(object):
 
     """Get processed stats using the psutil library."""
@@ -66,7 +56,6 @@ class GlancesProcesses(object):
         # Init stats
         self.auto_sort = True
         self._sort_key = 'cpu_percent'
-        self.allprocesslist = []
         self.processlist = []
         self.reset_processcount()
 
@@ -94,12 +83,27 @@ class GlancesProcesses(object):
         self.reset_max_values()
 
     def reset_processcount(self):
+        """Reset the global process count"""
         self.processcount = {'total': 0,
                              'running': 0,
                              'sleeping': 0,
                              'thread': 0,
                              'pid_max': None}
 
+    def update_processcount(self, plist):
+        """Update the global process count from the current processes list"""
+        # Update the maximum process ID (pid) number
+        self.processcount['pid_max'] = self.pid_max
+        # For each key in the processcount dict
+        # count the number of processes with the same status
+        for k in iterkeys(self.processcount):
+            self.processcount[k] = len(list(filter(lambda v: v['status'] is k,
+                                                   plist)))
+        # Compute thread
+        self.processcount['thread'] = sum(i['num_threads'] for i in plist)
+        # Compute total
+        self.processcount['total'] = len(plist)
+
     def enable(self):
         """Enable process stats."""
         self.disable_tag = False
@@ -224,110 +228,6 @@ class GlancesProcesses(object):
         for k in self._max_values_list:
             self._max_values[k] = 0.0
 
-    def __get_mandatory_stats(self, proc, procstat):
-        """
-        Get mandatory_stats: for all processes.
-        Needed for the sorting/filter step.
-
-        Stats grabbed inside this method:
-        * 'name', 'cpu_times', 'status', 'ppid'
-        * 'username', 'cpu_percent', 'memory_percent'
-        """
-        procstat['mandatory_stats'] = True
-
-        # Name, cpu_times, status and ppid stats are in the same /proc file
-        # Optimisation fir issue #958
-        try:
-            procstat.update(proc.as_dict(
-                attrs=['name', 'cpu_times', 'status', 'ppid'],
-                ad_value=''))
-        except (psutil.NoSuchProcess, psutil.AccessDenied):
-            # Try/catch for issue #432 (process no longer exist)
-            # Try/catch for issue #1120 (only see on Macos)
-            return None
-        else:
-            procstat['status'] = str(procstat['status'])[:1].upper()
-
-        try:
-            procstat.update(proc.as_dict(
-                attrs=['username', 'cpu_percent', 'memory_percent'],
-                ad_value=''))
-        except (psutil.NoSuchProcess, psutil.AccessDenied):
-            # Try/catch for issue #432 (process no longer exist)
-            return None
-
-        if procstat['cpu_percent'] == '' or procstat['memory_percent'] == '':
-            # Do not display process if we cannot get the basic
-            # cpu_percent or memory_percent stats
-            return None
-
-        # Compute the maximum value for cpu_percent and memory_percent
-        for k in self._max_values_list:
-            if procstat[k] > self.get_max_values(k):
-                self.set_max_values(k, procstat[k])
-
-        # Process command line (cached with internal cache)
-        if procstat['pid'] not in self.cmdline_cache:
-            # Patch for issue #391
-            try:
-                self.cmdline_cache[procstat['pid']] = proc.cmdline()
-            except (AttributeError, EnvironmentError, UnicodeDecodeError,
-                    psutil.AccessDenied, psutil.NoSuchProcess):
-                self.cmdline_cache[procstat['pid']] = ""
-        procstat['cmdline'] = self.cmdline_cache[procstat['pid']]
-
-        # Process IO
-        # procstat['io_counters'] is a list:
-        # [read_bytes, write_bytes, read_bytes_old, write_bytes_old, io_tag]
-        # If io_tag = 0 > Access denied (display "?")
-        # If io_tag = 1 > No access denied (display the IO rate)
-        # Availability: all platforms except macOS and Illumos/Solaris
-        try:
-            # Get the process IO counters
-            proc_io = proc.io_counters()
-            io_new = [proc_io.read_bytes, proc_io.write_bytes]
-        except (psutil.AccessDenied, psutil.NoSuchProcess, NotImplementedError):
-            # Access denied to process IO (no root account)
-            # NoSuchProcess (process die between first and second grab)
-            # Put 0 in all values (for sort) and io_tag = 0 (for display)
-            procstat['io_counters'] = [0, 0] + [0, 0]
-            io_tag = 0
-        except AttributeError:
-            return procstat
-        else:
-            # For IO rate computation
-            # Append saved IO r/w bytes
-            try:
-                procstat['io_counters'] = io_new + self.io_old[procstat['pid']]
-            except KeyError:
-                procstat['io_counters'] = io_new + [0, 0]
-            # then save the IO r/w bytes
-            self.io_old[procstat['pid']] = io_new
-            io_tag = 1
-
-        # Append the IO tag (for display)
-        procstat['io_counters'] += [io_tag]
-
-        return procstat
-
-    def __get_standard_stats(self, proc, procstat):
-        """
-        Get standard_stats: only for displayed processes.
-
-        Stats grabbed inside this method:
-        * nice and memory_info
-        """
-        procstat['standard_stats'] = True
-
-        # Process nice and memory_info (issue #926)
-        try:
-            procstat.update(
-                proc.as_dict(attrs=['nice', 'memory_info']))
-        except psutil.NoSuchProcess:
-            pass
-
-        return procstat
-
     def __get_extended_stats(self, proc, procstat):
         """
         Get extended stats, only for top processes (see issue #403).
@@ -377,175 +277,12 @@ class GlancesProcesses(object):
 
         return procstat
 
-    def __get_process_stats(self, proc,
-                            mandatory_stats=True,
-                            standard_stats=True,
-                            extended_stats=False):
-        """Get stats of a running processes."""
-        # Process ID (always)
-        procstat = proc.as_dict(attrs=['pid'])
-
-        if mandatory_stats:
-            procstat = self.__get_mandatory_stats(proc, procstat)
-
-        if procstat is not None and standard_stats:
-            procstat = self.__get_standard_stats(proc, procstat)
-
-        if procstat is not None and extended_stats and not self.disable_extended_tag:
-            procstat = self.__get_extended_stats(proc, procstat)
-
-        return procstat
-
     def update(self):
         """Update the processes stats."""
         # Reset the stats
         self.processlist = []
         self.reset_processcount()
 
-        # Do not process if disable tag is set
-        if self.disable_tag:
-            return
-
-        # Get the time since last update
-        time_since_update = getTimeSinceLastUpdate('process_disk')
-
-        # Reset the max dict
-        self.reset_max_values()
-
-        # Update the maximum process ID (pid) number
-        self.processcount['pid_max'] = self.pid_max
-
-        # Build an internal dict with only mandatories stats (sort keys)
-        processdict = {}
-        excluded_processes = set()
-        for proc in psutil.process_iter():
-            # Ignore kernel threads if needed
-            if self.no_kernel_threads and not WINDOWS and is_kernel_thread(proc):
-                continue
-
-            # If self.max_processes is None: Only retrieve mandatory stats
-            # Else: retrieve mandatory and standard stats
-            s = self.__get_process_stats(proc,
-                                         mandatory_stats=True,
-                                         standard_stats=self.max_processes is None)
-            # Check if s is note None (issue #879)
-            # ignore the 'idle' process on Windows and *BSD
-            # ignore the 'kernel_task' process on macOS
-            # waiting for upstream patch from psutil
-            if (s is None or
-                    BSD and s['name'] == 'idle' or
-                    WINDOWS and s['name'] == 'System Idle Process' or
-                    MACOS and s['name'] == 'kernel_task'):
-                continue
-            # Continue to the next process if it has to be filtered
-            if self._filter.is_filtered(s):
-                excluded_processes.add(proc)
-                continue
-
-            # Ok add the process to the list
-            processdict[proc] = s
-            # Update processcount (global statistics)
-            try:
-                self.processcount[str(proc.status())] += 1
-            except KeyError:
-                # Key did not exist, create it
-                try:
-                    self.processcount[str(proc.status())] = 1
-                except psutil.NoSuchProcess:
-                    pass
-            except psutil.NoSuchProcess:
-                pass
-            else:
-                self.processcount['total'] += 1
-            # Update thread number (global statistics)
-            try:
-                self.processcount['thread'] += proc.num_threads()
-            except Exception:
-                pass
-
-        if self._enable_tree:
-            self.process_tree = ProcessTreeNode.build_tree(processdict,
-                                                           self.sort_key,
-                                                           self.sort_reverse,
-                                                           self.no_kernel_threads,
-                                                           excluded_processes)
-
-            for i, node in enumerate(self.process_tree):
-                # Only retreive stats for visible processes (max_processes)
-                if self.max_processes is not None and i >= self.max_processes:
-                    break
-
-                # add standard stats
-                new_stats = self.__get_process_stats(node.process,
-                                                     mandatory_stats=False,
-                                                     standard_stats=True,
-                                                     extended_stats=False)
-                if new_stats is not None:
-                    node.stats.update(new_stats)
-
-                # Add a specific time_since_update stats for bitrate
-                node.stats['time_since_update'] = time_since_update
-
-        else:
-            # Process optimization
-            # Only retreive stats for visible processes (max_processes)
-            if self.max_processes is not None:
-                # Sort the internal dict and cut the top N (Return a list of tuple)
-                # tuple=key (proc), dict (returned by __get_process_stats)
-                try:
-                    processiter = sorted(iteritems(processdict),
-                                         key=lambda x: x[1][self.sort_key],
-                                         reverse=self.sort_reverse)
-                except (KeyError, TypeError) as e:
-                    logger.error("Cannot sort process list by {}: {}".format(self.sort_key, e))
-                    logger.error('{}'.format(listitems(processdict)[0]))
-                    # Fallback to all process (issue #423)
-                    processloop = iteritems(processdict)
-                    first = False
-                else:
-                    processloop = processiter[0:self.max_processes]
-                    first = True
-            else:
-                # Get all processes stats
-                processloop = iteritems(processdict)
-                first = False
-
-            for i in processloop:
-                # Already existing mandatory stats
-                procstat = i[1]
-                if self.max_processes is not None:
-                    # Update with standard stats
-                    # and extended stats but only for TOP (first) process
-                    s = self.__get_process_stats(i[0],
-                                                 mandatory_stats=False,
-                                                 standard_stats=True,
-                                                 extended_stats=first)
-                    if s is None:
-                        continue
-                    procstat.update(s)
-                # Add a specific time_since_update stats for bitrate
-                procstat['time_since_update'] = time_since_update
-                # Update process list
-                self.processlist.append(procstat)
-                # Next...
-                first = False
-
-        # Build the all processes list used by the AMPs
-        self.allprocesslist = [p for p in itervalues(processdict)]
-
-        # Clean internals caches if timeout is reached
-        if self.cache_timer.finished():
-            self.username_cache = {}
-            self.cmdline_cache = {}
-            # Restart the timer
-            self.cache_timer.reset()
-
-    def update_NEW(self):
-        """Update the processes stats."""
-        # Reset the stats
-        self.processlist = []
-        self.reset_processcount()
-
         # Do not process if disable tag is set
         if self.disable_tag:
             return
@@ -557,20 +294,84 @@ class GlancesProcesses(object):
         mandatories_attr = ['cmdline', 'cpu_percent', 'cpu_times',
                             'memory_info', 'memory_percent',
                             'name', 'nice', 'pid',
-                            'ppid', 'status', 'username']
-        # io_counters is not available on macOS and Illumos/Solaris
-        if not MACOS and not SUNOS:
+                            'ppid', 'status', 'username',
+                            'status', 'num_threads', 'gids']
+        # io_counters availability: Linux, BSD, Windows, AIX
+        if LINUX or BSD or WINDOWS:
             mandatories_attr += ['io_counters']
-        # and build the processes stats list
-        self.processlist = [p.info for p in sorted(psutil.process_iter(attrs=mandatories_attr,
-                                                                       ad_value=None),
-                                                   key=lambda p: p.info['cpu_percent'])]
 
-        # Update the maximum process ID (pid) number
-        self.processcount['pid_max'] = self.pid_max
+        # and build the processes stats list
+        self.processlist = [p.info for p in psutil.process_iter(attrs=mandatories_attr,
+                                                                ad_value=None)
+                            # OS specifics processes filter
+                            if not (BSD and p.info['name'] == 'idle') and
+                            not (WINDOWS and p.info['name'] == 'System Idle Process') and
+                            not (MACOS and p.info['name'] == 'kernel_task') and
+                            # Kernel threads filter
+                            not (self.no_kernel_threads and LINUX and p.info['gids'].real == 0) and
+                            # User filter
+                            not (self._filter.is_filtered(p.info))
+                            ]
+
+        # Sort the processes list by the current sort_key
+        self.processlist = sorted(self.processlist,
+                                  key=lambda p: p[self.sort_key],
+                                  reverse=True)
+
+        # Update the processcount
+        self.update_processcount(self.processlist)
 
         # Loop over processes and add metadata
+        first = True
         for proc in self.processlist:
+            if first and not self.disable_extended_tag:
+                # Get extended stats, only for top processes (see issue #403).
+                # - cpu_affinity (Linux, Windows, FreeBSD)
+                # - ionice (Linux and Windows > Vista)
+                # - memory_full_info (Linux)
+                # - num_ctx_switches (not available on Illumos/Solaris)
+                # - num_fds (Unix-like)
+                # - num_handles (Windows)
+                # - num_threads (not available on *BSD)
+                # - memory_maps (only swap, Linux)
+                #   https://www.cyberciti.biz/faq/linux-which-process-is-using-swap/
+                # - connections (TCP and UDP)
+                extended = {}
+                try:
+                    top_process = psutil.Process(proc['pid'])
+                    extended_stats = ['cpu_affinity', 'ionice',
+                                      'memory_full_info', 'num_ctx_switches',
+                                      'num_fds', 'num_threads']
+                    if WINDOWS:
+                        extended_stats += ['num_handles']
+                    extended = top_process.as_dict(attrs=extended_stats)
+                    # !!! TODO
+                    # if LINUX:
+                    #     try:
+                    #         procstat['memory_swap'] = sum([v.swap for v in proc.memory_maps()])
+                    #     except psutil.NoSuchProcess:
+                    #         pass
+                    #     except (psutil.AccessDenied, TypeError, NotImplementedError):
+                    #         # NotImplementedError: /proc/${PID}/smaps file doesn't exist
+                    #         # on kernel < 2.6.14 or CONFIG_MMU kernel configuration option
+                    #         # is not enabled (see psutil #533/glances #413).
+                    #         # XXX: Remove TypeError once we'll drop psutil < 3.0.0.
+                    #         procstat['memory_swap'] = None
+                    # try:
+                    #     procstat['tcp'] = len(proc.connections(kind="tcp"))
+                    #     procstat['udp'] = len(proc.connections(kind="udp"))
+                    # except psutil.AccessDenied:
+                    #     procstat['tcp'] = None
+                    #     procstat['udp'] = None
+                except (psutil.NoSuchProcess, ValueError, AttributeError) as e:
+                    logger.error('Can not grab extended stats ({})'.format(e))
+                    extended['extended_stats'] = False
+                else:
+                    logger.debug('Grab extended stats for process {}'.format(proc['pid']))
+                    extended['extended_stats'] = True
+                proc.update(extended)
+            first = False
+
             # Time since last update (for disk_io rate computation)
             proc['time_since_update'] = time_since_update
 
@@ -598,23 +399,18 @@ class GlancesProcesses(object):
             else:
                 proc['io_counters'] = [0, 0] + [0, 0]
                 io_tag = 0
-
             # Append the IO tag (for display)
             proc['io_counters'] += [io_tag]
 
-        # Compute the maximum value for keys in self._max_values_list
-        # Compute max
+        # Compute the maximum value for keys in self._max_values_list (CPU, MEM)
         for k in self._max_values_list:
-            self.set_max_values(k, max(i[k] for i in self.processlist))
+            if self.processlist != []:
+                self.set_max_values(k, max(i[k] for i in self.processlist))
 
     def getcount(self):
         """Get the number of processes."""
         return self.processcount
 
-    def getalllist(self):
-        """Get the allprocesslist."""
-        return self.allprocesslist
-
     def getlist(self, sortedby=None):
         """Get the processlist."""
         return self.processlist