avocado.sysinfo: Introduce system information profilers.

Sysinfo module now allows to define profilers, which are programs
defined in command line, to start as a background processes and log
the output in a proper directory. Example: vmstat 1.

The profilers are started before a job execution and then, stopped at
the end of the job. The output is stored inside `sysinfo/profile/*`
subdirectory, in job result directory.

The profilers can be customized in `avocado.conf`, section
`sysinfo.collect`.  The default configuration define
profilers as the following: `journalctl -f` and `vmstat 1`.

This feature is turned off by default.
Signed-off-by: NRudá Moura <rmoura@redhat.com>

avocado/sysinfo.py

@@ -364,7 +364,7 @@ class SysInfo(object):
     * end_job
     """
 
-    def __init__(self, basedir=None, log_packages=None):
+    def __init__(self, basedir=None, log_packages=None, profilers=None):
         """
         Set sysinfo loggables.
 
@@ -373,6 +373,8 @@ class SysInfo(object):
                              logging packages is a costly operation). If not
                              given explicitly, tries to look in the config
                              files, and if not found, defaults to False.
+        :param profilers: Wether to use the profiler. If not given explicitly,
+                          tries to look in the config files.
         """
         if basedir is None:
             basedir = utils.path.init_dir(os.getcwd(), 'sysinfo')
@@ -388,6 +390,30 @@ class SysInfo(object):
         else:
             self.log_packages = log_packages
 
+        if profilers is None:
+            self.profiler = settings.get_value('sysinfo.collect',
+                                               'profiler',
+                                               key_type='bool',
+                                               default=False)
+            profiler_commands = settings.get_value('sysinfo.collect',
+                                                   'profiler_commands',
+                                                   key_type='str',
+                                                   default='')
+        else:
+            self.profiler = True
+            profiler_commands = profilers
+
+        self.profiler_commands = [x for x in profiler_commands.split(':') if x.strip()]
+        log.info('Profilers declared: %s', self.profiler_commands)
+        if not self.profiler_commands:
+            self.profiler = False
+
+        if self.profiler is False:
+            if not self.profiler_commands:
+                log.info('Profiler disabled: no profiler commands configured')
+            else:
+                log.info('Profiler disabled')
+
         self.start_job_loggables = set()
         self.end_job_loggables = set()
 
@@ -427,6 +453,10 @@ class SysInfo(object):
         return syslog_watcher
 
     def _set_loggables(self):
+        if self.profiler:
+            for cmd in self.profiler_commands:
+                self.start_job_loggables.add(Daemon(cmd))
+
         for cmd in _DEFAULT_COMMANDS_START_JOB:
             self.start_job_loggables.add(Command(cmd))
 
@@ -539,7 +569,10 @@ class SysInfo(object):
         Logging hook called whenever a job starts.
         """
         for log in self.start_job_loggables:
-            log.run(pre_dir)
+            if isinstance(log, Daemon):  # log daemons in profile directory
+                log.run(self.profile_dir)
+            else:
+                log.run(self.pre_dir)
 
         if self.log_packages:
             self._log_installed_packages(self.pre_dir)
@@ -552,7 +585,8 @@ class SysInfo(object):
             log.run(self.post_dir)
         # Stop daemon(s) started previously
         for log in self.start_job_loggables:
-            log.run(post_dir)
+            if isinstance(log, Daemon):
+                log.stop()
 
         if self.log_packages:
             self._log_modified_packages(self.post_dir)

etc/avocado/avocado.conf

@@ -7,3 +7,5 @@ logs_dir = ~/avocado/job-results
 [sysinfo.collect]
 enabled = True
 installed_packages = False
+profiler = False
+profiler_commands = vmstat 1:journalctl -f

selftests/all/functional/avocado/multiplex_tests.py

@@ -41,7 +41,7 @@ class MultiplexTests(unittest.TestCase):
                                     'The multiplexed job log output has less '
                                     'lines than expected\n%s' %
                                     "".join(job_log_lines))
-            self.assertLess(lines_output, expected_lines * 1.1,
+            self.assertLess(lines_output, expected_lines * 1.2,
                             'The multiplexed job log output has more '
                             'lines than expected\n%s'
                             % "".join(job_log_lines))

selftests/all/unit/avocado/sysinfo_unittest.py

@@ -60,8 +60,8 @@ class SysinfoTest(unittest.TestCase):
         sysinfo_logger = sysinfo.SysInfo(basedir=jobdir)
         sysinfo_logger.start_job_hook()
         self.assertTrue(os.path.isdir(jobdir))
-        self.assertEqual(len(os.listdir(jobdir)), 1,
-                         "Job does not have 'pre' dir")
+        self.assertGreaterEqual(len(os.listdir(jobdir)), 1,
+                                "Job does not have 'pre' dir")
         job_predir = os.path.join(jobdir, 'pre')
         self.assertTrue(os.path.isdir(job_predir))
         self.assertGreater(len(os.listdir(job_predir)), 0,
@@ -77,16 +77,16 @@ class SysinfoTest(unittest.TestCase):
         sysinfo_logger = sysinfo.SysInfo(basedir=testdir)
         sysinfo_logger.start_test_hook()
         self.assertTrue(os.path.isdir(testdir))
-        self.assertEqual(len(os.listdir(testdir)), 1,
-                         "Test does not have 'pre' dir")
+        self.assertGreaterEqual(len(os.listdir(testdir)), 1,
+                                "Test does not have 'pre' dir")
         test_predir = os.path.join(testdir, 'pre')
         self.assertTrue(os.path.isdir(test_predir))
         # By default, there are no pre test files
         self.assertEqual(len(os.listdir(test_predir)), 0,
                          "Test pre dir is not empty")
         sysinfo_logger.end_test_hook()
-        self.assertEqual(len(os.listdir(testdir)), 2,
-                         "Test does not have 'pre' dir")
+        self.assertGreaterEqual(len(os.listdir(testdir)), 2,
+                                "Test does not have 'pre' dir")
         job_postdir = os.path.join(testdir, 'post')
         self.assertTrue(os.path.isdir(job_postdir))
         # By default, there are no post test files