Merge pull request #542 from clebergnu/sysinfo_configurable_v2

Sysinfo: make it configurable [v2]

avocado.spec

 Summary: Avocado Test Framework
 Name: avocado
 Version: 0.21.0
-Release: 5%{?dist}
+Release: 6%{?dist}
 License: GPLv2
 Group: Development/Tools
 URL: http://avocado-framework.github.io/
@@ -54,8 +54,12 @@ selftests/run selftests/all/unit
 %doc README.rst LICENSE
 %dir /etc/avocado
 %dir /etc/avocado/conf.d
+%dir /etc/avocado/sysinfo
 %config(noreplace)/etc/avocado/avocado.conf
 %config(noreplace)/etc/avocado/conf.d/README
+%config(noreplace)/etc/avocado/sysinfo/commands
+%config(noreplace)/etc/avocado/sysinfo/files
+%config(noreplace)/etc/avocado/sysinfo/profilers
 %{python_sitelib}/avocado*
 %{_bindir}/avocado
 %{_bindir}/avocado-rest-client
@@ -99,6 +103,9 @@ examples of how to write tests on your own.
 %{_datadir}/avocado/api
 
 %changelog
+* Mon Apr 13 2015 Cleber Rosa <cleber@redhat.com> - 0.21.0-6
+- Added sysinfo configuration files
+
 * Sat Mar 28 2015 Cleber Rosa <cleber@redhat.com> - 0.21.0-5
 - Change the way man pages are built, now using Makefile targets
 - Reorganized runtime and build requirements

avocado/sysinfo.py

@@ -31,68 +31,10 @@ from avocado.settings import settings
 log = logging.getLogger("avocado.sysinfo")
 
 
-_DEFAULT_COMMANDS_START_JOB = ["df -mP",
-                               "dmesg -c",
-                               "uname -a",
-                               "lspci -vvnn",
-                               "gcc --version",
-                               "ld --version",
-                               "mount",
-                               "hostname",
-                               "uptime",
-                               "dmidecode",
-                               "ifconfig -a",
-                               "brctl show",
-                               "ip link",
-                               "numactl --hardware show",
-                               "lscpu",
-                               "fdisk -l"]
-
-_DEFAULT_COMMANDS_END_JOB = _DEFAULT_COMMANDS_START_JOB
-
-_DEFAULT_FILES_START_JOB = ["/proc/cmdline",
-                            "/proc/mounts",
-                            "/proc/pci",
-                            "/proc/meminfo",
-                            "/proc/slabinfo",
-                            "/proc/version",
-                            "/proc/cpuinfo",
-                            "/proc/modules",
-                            "/proc/interrupts",
-                            "/proc/partitions",
-                            "/sys/kernel/debug/sched_features",
-                            "/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor",
-                            "/sys/devices/system/clocksource/clocksource0/current_clocksource"]
-
-_DEFAULT_FILES_END_JOB = _DEFAULT_FILES_START_JOB
-
-_DEFAULT_COMMANDS_START_TEST = []
-
-_DEFAULT_COMMANDS_END_TEST = []
-
-_DEFAULT_FILES_START_TEST = []
-
-_DEFAULT_FILES_END_TEST = []
-
-_DEFAULT_COMMANDS_START_ITERATION = []
-_DEFAULT_COMMANDS_END_ITERATION = ["/proc/schedstat",
-                                   "/proc/meminfo",
-                                   "/proc/slabinfo",
-                                   "/proc/interrupts",
-                                   "/proc/buddyinfo"]
-
-_DEFAULT_FILES_START_ITERATION = []
-_DEFAULT_FILES_END_ITERATION = ["/proc/schedstat",
-                                "/proc/meminfo",
-                                "/proc/slabinfo",
-                                "/proc/interrupts",
-                                "/proc/buddyinfo"]
-
-
-class Loggable(object):
+class Collectible(object):
 
     """
-    Abstract class for representing all things "loggable" by sysinfo.
+    Abstract class for representing collectibles by sysinfo.
     """
 
     def __init__(self, logf):
@@ -100,7 +42,7 @@ class Loggable(object):
 
     def readline(self, logdir):
         """
-        Read one line of the loggable object.
+        Read one line of the collectible object.
 
         :param logdir: Path to a log directory.
         """
@@ -111,10 +53,10 @@ class Loggable(object):
             return ""
 
 
-class Logfile(Loggable):
+class Logfile(Collectible):
 
     """
-    Loggable system file.
+    Collectible system file.
 
     :param path: Path to the log file.
     :param logf: Basename of the file where output is logged (optional).
@@ -134,7 +76,7 @@ class Logfile(Loggable):
     def __eq__(self, other):
         if isinstance(other, Logfile):
             return (self.path, self.logf) == (other.path, other.logf)
-        elif isinstance(other, Loggable):
+        elif isinstance(other, Collectible):
             return False
         return NotImplemented
 
@@ -160,10 +102,10 @@ class Logfile(Loggable):
                 log.debug("Not logging %s (lack of permissions)", self.path)
 
 
-class Command(Loggable):
+class Command(Collectible):
 
     """
-    Loggable command.
+    Collectible command.
 
     :param cmd: String with the command.
     :param logf: Basename of the file where output is logged (optional).
@@ -185,7 +127,7 @@ class Command(Loggable):
     def __eq__(self, other):
         if isinstance(other, Command):
             return (self.cmd, self.logf) == (other.cmd, other.logf)
-        elif isinstance(other, Loggable):
+        elif isinstance(other, Collectible):
             return False
         return NotImplemented
 
@@ -225,7 +167,7 @@ class Command(Loggable):
 class Daemon(Command):
 
     """
-    Loggable daemon.
+    Collectible daemon.
 
     :param cmd: String with the daemon command.
     :param logf: Basename of the file where output is logged (optional).
@@ -261,7 +203,7 @@ class Daemon(Command):
         return retcode
 
 
-class LogWatcher(Loggable):
+class LogWatcher(Collectible):
 
     """
     Keep track of the contents of a log file in another compressed file.
@@ -299,7 +241,7 @@ class LogWatcher(Loggable):
     def __eq__(self, other):
         if isinstance(other, Logfile):
             return (self.path, self.logf) == (other.path, other.logf)
-        elif isinstance(other, Loggable):
+        elif isinstance(other, Collectible):
             return False
         return NotImplemented
 
@@ -358,27 +300,24 @@ class SysInfo(object):
 
     * start_job
     * start_test
-    * start_iteration
-    * end_iteration
     * end_test
     * end_job
     """
 
-    def __init__(self, basedir=None, log_packages=None, profilers=None):
+    def __init__(self, basedir=None, log_packages=None, profiler=None):
         """
-        Set sysinfo loggables.
+        Set sysinfo collectibles.
 
         :param basedir: Base log dir where sysinfo files will be located.
         :param log_packages: Whether to log system packages (optional because
                              logging packages is a costly operation). If not
                              given explicitly, tries to look in the config
                              files, and if not found, defaults to False.
-        :param profilers: Wether to use the profiler. If not given explicitly,
-                          tries to look in the config files.
+        :param profiler: Wether to use the profiler. If not given explicitly,
+                         tries to look in the config files.
         """
         if basedir is None:
-            basedir = utils.path.init_dir(os.getcwd(), 'sysinfo')
-
+            basedir = utils.path.init_dir('sysinfo')
         self.basedir = basedir
 
         self._installed_pkgs = None
@@ -390,51 +329,61 @@ class SysInfo(object):
         else:
             self.log_packages = log_packages
 
-        if profilers is None:
+        commands_file = settings.get_value('sysinfo.collectibles',
+                                           'commands',
+                                           key_type='str',
+                                           default='')
+        log.info('Commands configured by file: %s', commands_file)
+        self.commands = utils.genio.read_all_lines(commands_file)
+
+        files_file = settings.get_value('sysinfo.collectibles',
+                                        'files',
+                                        key_type='str',
+                                        default='')
+        log.info('Files configured by file: %s', files_file)
+        self.files = utils.genio.read_all_lines(files_file)
+
+        if profiler is None:
             self.profiler = settings.get_value('sysinfo.collect',
                                                'profiler',
                                                key_type='bool',
                                                default=False)
-            profiler_commands = settings.get_value('sysinfo.collect',
-                                                   'profiler_commands',
-                                                   key_type='str',
-                                                   default='')
         else:
-            self.profiler = True
-            profiler_commands = profilers
+            self.profiler = profiler
+
+        profiler_file = settings.get_value('sysinfo.collectibles',
+                                           'profilers',
+                                           key_type='str',
+                                           default='')
+        self.profilers = utils.genio.read_all_lines(profiler_file)
 
-        self.profiler_commands = [x for x in profiler_commands.split(':') if x.strip()]
-        log.info('Profilers declared: %s', self.profiler_commands)
-        if not self.profiler_commands:
+        log.info('Profilers configured by file: %s', profiler_file)
+        log.info('Profilers declared: %s', self.profilers)
+        if not self.profilers:
             self.profiler = False
 
         if self.profiler is False:
-            if not self.profiler_commands:
+            if not self.profilers:
                 log.info('Profiler disabled: no profiler commands configured')
             else:
                 log.info('Profiler disabled')
 
-        self.start_job_loggables = set()
-        self.end_job_loggables = set()
-
-        self.start_test_loggables = set()
-        self.end_test_loggables = set()
+        self.start_job_collectibles = set()
+        self.end_job_collectibles = set()
 
-        self.start_iteration_loggables = set()
-        self.end_iteration_loggables = set()
+        self.start_test_collectibles = set()
+        self.end_test_collectibles = set()
 
-        self.hook_mapping = {'start_job': self.start_job_loggables,
-                             'end_job': self.end_job_loggables,
-                             'start_test': self.start_test_loggables,
-                             'end_test': self.end_test_loggables,
-                             'start_iteration': self.start_iteration_loggables,
-                             'end_iteration': self.end_iteration_loggables}
+        self.hook_mapping = {'start_job': self.start_job_collectibles,
+                             'end_job': self.end_job_collectibles,
+                             'start_test': self.start_test_collectibles,
+                             'end_test': self.end_test_collectibles}
 
         self.pre_dir = utils.path.init_dir(self.basedir, 'pre')
         self.post_dir = utils.path.init_dir(self.basedir, 'post')
         self.profile_dir = utils.path.init_dir(self.basedir, 'profile')
 
-        self._set_loggables()
+        self._set_collectibles()
 
     def _get_syslog_watcher(self):
         syslog_watcher = None
@@ -452,93 +401,65 @@ class SysInfo(object):
 
         return syslog_watcher
 
-    def _set_loggables(self):
+    def _set_collectibles(self):
         if self.profiler:
-            for cmd in self.profiler_commands:
-                self.start_job_loggables.add(Daemon(cmd))
-
-        for cmd in _DEFAULT_COMMANDS_START_JOB:
-            self.start_job_loggables.add(Command(cmd))
-
-        for cmd in _DEFAULT_COMMANDS_END_JOB:
-            self.end_job_loggables.add(Command(cmd))
-
-        for filename in _DEFAULT_FILES_START_JOB:
-            self.start_job_loggables.add(Logfile(filename))
+            for cmd in self.profilers:
+                self.start_job_collectibles.add(Daemon(cmd))
 
-        for filename in _DEFAULT_FILES_END_JOB:
-            self.end_job_loggables.add(Logfile(filename))
+        for cmd in self.commands:
+            self.start_job_collectibles.add(Command(cmd))
+            self.end_job_collectibles.add(Command(cmd))
 
-        for cmd in _DEFAULT_COMMANDS_START_TEST:
-            self.start_test_loggables.add(Command(cmd))
-
-        for cmd in _DEFAULT_COMMANDS_END_TEST:
-            self.end_test_loggables.add(Command(cmd))
+        for filename in self.files:
+            self.start_job_collectibles.add(Logfile(filename))
+            self.end_job_collectibles.add(Logfile(filename))
 
         # As the system log path is not standardized between distros,
         # we have to probe and find out the correct path.
         try:
-            self.end_test_loggables.add(self._get_syslog_watcher())
+            self.end_test_collectibles.add(self._get_syslog_watcher())
         except ValueError, details:
             log.info(details)
 
-        for filename in _DEFAULT_FILES_START_TEST:
-            self.start_test_loggables.add(Logfile(filename))
-
-        for filename in _DEFAULT_FILES_END_TEST:
-            self.end_test_loggables.add(Logfile(filename))
-
-        for cmd in _DEFAULT_COMMANDS_START_ITERATION:
-            self.start_iteration_loggables.add(Command(cmd))
-
-        for cmd in _DEFAULT_COMMANDS_END_ITERATION:
-            self.end_iteration_loggables.add(Command(cmd))
-
-        for filename in _DEFAULT_FILES_START_ITERATION:
-            self.start_iteration_loggables.add(Logfile(filename))
-
-        for filename in _DEFAULT_FILES_END_ITERATION:
-            self.end_iteration_loggables.add(Logfile(filename))
-
-    def _get_loggables(self, hook):
-        loggables = self.hook_mapping.get(hook)
-        if loggables is None:
+    def _get_collectibles(self, hook):
+        collectibles = self.hook_mapping.get(hook)
+        if collectibles is None:
             raise ValueError('Incorrect hook, valid hook names: %s' %
                              self.hook_mapping.keys())
-        return loggables
+        return collectibles
 
     def add_cmd(self, cmd, hook):
         """
-        Add a command loggable.
+        Add a command collectible.
 
         :param cmd: Command to log.
         :param hook: In which hook this cmd should be logged (start job, end
-                     job, start iteration, end iteration).
+                     job).
         """
-        loggables = self._get_loggables(hook)
-        loggables.add(Command(cmd))
+        collectibles = self._get_collectibles(hook)
+        collectibles.add(Command(cmd))
 
     def add_file(self, filename, hook):
         """
-        Add a system file loggable.
+        Add a system file collectible.
 
         :param filename: Path to the file to be logged.
         :param hook: In which hook this file should be logged (start job, end
-                     job, start iteration, end iteration).
+                     job).
         """
-        loggables = self._get_loggables(hook)
-        loggables.add(Logfile(filename))
+        collectibles = self._get_collectibles(hook)
+        collectibles.add(Logfile(filename))
 
     def add_watcher(self, filename, hook):
         """
-        Add a system file watcher loggable.
+        Add a system file watcher collectible.
 
         :param filename: Path to the file to be logged.
         :param hook: In which hook this watcher should be logged (start job, end
-                     job, start iteration, end iteration).
+                     job).
         """
-        loggables = self._get_loggables(hook)
-        loggables.add(LogWatcher(filename))
+        collectibles = self._get_collectibles(hook)
+        collectibles.add(LogWatcher(filename))
 
     def _get_installed_packages(self):
         sm = software_manager.SoftwareManager()
@@ -568,7 +489,7 @@ class SysInfo(object):
         """
         Logging hook called whenever a job starts.
         """
-        for log in self.start_job_loggables:
+        for log in self.start_job_collectibles:
             if isinstance(log, Daemon):  # log daemons in profile directory
                 log.run(self.profile_dir)
             else:
@@ -581,10 +502,10 @@ class SysInfo(object):
         """
         Logging hook called whenever a job finishes.
         """
-        for log in self.end_job_loggables:
+        for log in self.end_job_collectibles:
             log.run(self.post_dir)
         # Stop daemon(s) started previously
-        for log in self.start_job_loggables:
+        for log in self.start_job_collectibles:
             if isinstance(log, Daemon):
                 log.stop()
 
@@ -595,7 +516,7 @@ class SysInfo(object):
         """
         Logging hook called before a test starts.
         """
-        for log in self.start_test_loggables:
+        for log in self.start_test_collectibles:
             log.run(self.pre_dir)
 
         if self.log_packages:
@@ -605,26 +526,12 @@ class SysInfo(object):
         """
         Logging hook called after a test finishes.
         """
-        for log in self.end_test_loggables:
+        for log in self.end_test_collectibles:
             log.run(self.post_dir)
 
         if self.log_packages:
             self._log_modified_packages(self.post_dir)
 
-    def start_iteration_hook(self):
-        """
-        Logging hook called before a test iteration
-        """
-        for log in self.start_iteration_loggables:
-            log.run(self.pre_dir)
-
-    def end_iteration_hook(self, test, iteration=None):
-        """
-        Logging hook called after a test iteration
-        """
-        for log in self.end_iteration_loggables:
-            log.run(self.post_dir)
-
 
 def collect_sysinfo(args):
     """

avocado/utils/genio.py

@@ -126,6 +126,32 @@ def read_one_line(filename):
     return line
 
 
+def read_all_lines(filename):
+    """
+    Return all lines of a given file
+
+    This utility method returns an empty list in any error scenario,
+    that is, it doesn't attempt to identify error paths and raise
+    appropriate exceptions. It does exactly the opposite to that.
+
+    This should be used when it's fine or desirable to have an empty
+    set of lines if a file is missing or is unreadable.
+
+    :param filename: Path to the file.
+    :type filename: str
+
+    :return: all lines of the file as list
+    :rtype: list
+    """
+    contents = []
+    try:
+        with open(filename, 'r') as file_obj:
+            contents = [line.rstrip('\n') for line in file_obj.readlines()]
+    except:
+        pass
+    return contents
+
+
 def write_file(filename, data):
     """
     Write data to a file.

etc/avocado/avocado.conf

@@ -15,8 +15,14 @@ enabled = True
 installed_packages = False
 # Whether to run certain commands in bg to give extra job debug information
 profiler = False
-# Commands to run in bg (colon separated)
-profiler_commands = vmstat 1:journalctl -f
+
+[sysinfo.collectibles]
+# File with list of commands that will be executed and have their output collected
+commands = /etc/avocado/sysinfo/commands
+# File with list of files that will be collected verbatim
+files = /etc/avocado/sysinfo/files
+# File with list of commands that will run alongside the job/test
+profilers = /etc/avocado/sysinfo/profilers
 
 [runner.output]
 # Whether to display colored output in terminals that support it

etc/avocado/sysinfo/commands

+df -mP
+dmesg -c
+uname -a
+lspci -vvnn
+gcc --version
+ld --version
+mount
+hostname
+uptime
+dmidecode
+ifconfig -a
+brctl show
+ip link
+numactl --hardware show
+lscpu
+fdisk -l

etc/avocado/sysinfo/files

+/proc/cmdline
+/proc/mounts
+/proc/pci
+/proc/meminfo
+/proc/slabinfo
+/proc/version
+/proc/cpuinfo
+/proc/modules
+/proc/interrupts
+/proc/partitions
+/sys/kernel/debug/sched_features
+/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor
+/sys/devices/system/clocksource/clocksource0/current_clocksource

etc/avocado/sysinfo/profilers

+vmstat 1
+journalctl -f

selftests/all/functional/avocado/sysinfo_tests.py

@@ -40,8 +40,8 @@ class SysInfoTest(unittest.TestCase):
         self.assertGreater(len(os.listdir(sysinfo_dir)), 0, msg)
         for hook in ('pre', 'post'):
             sysinfo_subdir = os.path.join(sysinfo_dir, hook)
-            msg = 'The sysinfo/%s subdirectory is empty:\n%s' % (hook, result)
-            self.assertGreater(len(os.listdir(sysinfo_subdir)), 0, msg)
+            msg = 'The sysinfo/%s subdirectory does not exist:\n%s' % (hook, result)
+            self.assertTrue(os.path.exists(sysinfo_subdir), msg)
 
     def test_sysinfo_disabled(self):
         os.chdir(basedir)

selftests/all/unit/avocado/sysinfo_unittest.py

@@ -69,13 +69,9 @@ class SysinfoTest(unittest.TestCase):
                                 "Job does not have 'pre' dir")
         job_predir = os.path.join(jobdir, 'pre')
         self.assertTrue(os.path.isdir(job_predir))
-        self.assertGreater(len(os.listdir(job_predir)), 0,
-                           "Job pre dir is empty")
         sysinfo_logger.end_job_hook()
         job_postdir = os.path.join(jobdir, 'post')
         self.assertTrue(os.path.isdir(job_postdir))
-        self.assertGreater(len(os.listdir(job_postdir)), 0,
-                           "Job post dir is empty")
 
     def test_logger_test_hooks(self):
         testdir = os.path.join(self.tmpdir, 'job', 'test1')

setup.py

@@ -61,6 +61,9 @@ def get_data_files():
     data_files = [(get_dir(['etc', 'avocado']), ['etc/avocado/avocado.conf'])]
     data_files += [(get_dir(['etc', 'avocado', 'conf.d']),
                     ['etc/avocado/conf.d/README'])]
+    data_files += [(get_dir(['etc', 'avocado', 'sysinfo']),
+                    ['etc/avocado/sysinfo/commands', 'etc/avocado/sysinfo/files',
+                     'etc/avocado/sysinfo/profilers'])]
     data_files += [(get_tests_dir(), glob.glob('examples/tests/*.py'))]
     for data_dir in glob.glob('examples/tests/*.data'):
         fmt_str = '%s/*' % data_dir