Merging pull request 1382

* https://github.com/avocado-framework/avocado:
  Introduce avocado job diff plugin
  avocado.core.replay record command line

avocado/core/job.py

@@ -489,7 +489,7 @@ class Job(object):
         self._start_sysinfo()
 
         self._log_job_debug_info(mux)
-        replay.record(self.args, self.logdir, mux, self.urls)
+        replay.record(self.args, self.logdir, mux, self.urls, sys.argv)
         replay_map = getattr(self.args, 'replay_map', None)
         summary = self.test_runner.run_suite(self.test_suite, mux, self.timeout,
                                              replay_map,

avocado/core/replay.py

@@ -30,13 +30,14 @@ Record/retrieve job information for job replay
 """
 
 
-def record(args, logdir, mux, urls=None):
+def record(args, logdir, mux, urls=None, cmdline=None):
     replay_dir = path.init_dir(logdir, 'replay')
     path_cfg = os.path.join(replay_dir, 'config')
     path_urls = os.path.join(replay_dir, 'urls')
     path_mux = os.path.join(replay_dir, 'multiplex')
     path_pwd = os.path.join(replay_dir, 'pwd')
     path_args = os.path.join(replay_dir, 'args')
+    path_cmdline = os.path.join(replay_dir, 'cmdline')
 
     if urls:
         with open(path_urls, 'w') as f:
@@ -54,6 +55,20 @@ def record(args, logdir, mux, urls=None):
     with open(path_args, 'w') as f:
         pickle.dump(args.__dict__, f, pickle.HIGHEST_PROTOCOL)
 
+    with open(path_cmdline, 'w') as f:
+        f.write('%s' % cmdline)
+
+
+def retrieve_cmdline(resultsdir):
+    recorded_cmdline = os.path.join(resultsdir, "replay", "cmdline")
+    if not os.path.exists(recorded_cmdline):
+        return None
+
+    with open(recorded_cmdline, 'r') as f:
+        cmdline = f.read()
+
+    return ast.literal_eval(cmdline)
+
 
 def retrieve_pwd(resultsdir):
     recorded_pwd = os.path.join(resultsdir, "replay", "pwd")

avocado/plugins/diff.py

+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See LICENSE for more details.
+#
+# Copyright: Red Hat Inc. 2016
+# Author: Amador Pahim <apahim@redhat.com>
+
+"""
+Job Diff
+"""
+
+from __future__ import absolute_import
+import argparse
+import json
+import logging
+import os
+import subprocess
+import sys
+import tempfile
+
+from difflib import unified_diff, HtmlDiff
+
+from avocado.core import exit_codes
+from avocado.core import replay
+from avocado.core import output
+
+from avocado.core.plugin_interfaces import CLICmd
+from avocado.core.settings import settings
+
+
+LOG = logging.getLogger("avocado.app")
+
+
+class Diff(CLICmd):
+
+    """
+    Implements the avocado 'diff' subcommand
+    """
+
+    name = 'diff'
+    description = 'Shows the difference between 2 jobs.'
+
+    def __init__(self):
+        self.term = output.TERM_SUPPORT
+        self.std_diff_output = True
+
+    def configure(self, parser):
+        """
+        Add the subparser for the diff action.
+
+        :param parser: Main test runner parser.
+        """
+        parser = super(Diff, self).configure(parser)
+
+        parser.add_argument("jobids",
+                            default=[], nargs=2,
+                            metavar="<JOB>",
+                            help='A job reference, identified by a (partial) '
+                            'unique ID (SHA1) or test results directory.')
+
+        parser.add_argument('--html', type=str,
+                            metavar='FILE',
+                            help='Enable HTML output to the FILE where the '
+                            'result should be written.')
+
+        parser.add_argument('--open-browser',
+                            action='store_true',
+                            default=False,
+                            help='Generate and open a HTML report in your '
+                            'preferred browser. If no --html file is '
+                            'provided, create a temporary file.')
+
+        parser.add_argument('--diff-filter',
+                            dest='diff_filter',
+                            type=self._validate_filters,
+                            default=['cmdline', 'time', 'variants',
+                                     'results', 'config', 'sysinfo'],
+                            help='Comma separated filter of diff sections: '
+                            '(no)cmdline,(no)time,(no)variants,(no)results,\n'
+                            '(no)config,(no)sysinfo (defaults to all '
+                            'enabled).')
+
+        parser.add_argument('--paginator',
+                            choices=('on', 'off'), default='on',
+                            help='Turn the paginator on/off. '
+                            'Current: %(default)s')
+
+        parser.add_argument('--create-reports', action='store_true',
+                            help='Create temporary files with job reports '
+                            '(to be used by other diff tools)')
+
+        parser.epilog = 'By default, a textual diff report is generated '\
+                        'in the standard output.'
+
+    def run(self, args):
+        job1_dir, job1_id = self._setup_job(args.jobids[0])
+        job2_dir, job2_id = self._setup_job(args.jobids[1])
+
+        job1_data = self._get_job_data(job1_dir)
+        job2_data = self._get_job_data(job2_dir)
+
+        report_header = 'Avocado Job Report\n'
+        job1_results = [report_header]
+        job2_results = [report_header]
+
+        if 'cmdline' in args.diff_filter:
+            cmdline1 = self._get_command_line(job1_dir)
+            cmdline2 = self._get_command_line(job2_dir)
+
+            if str(cmdline1) != str(cmdline2):
+                command_line_header = ['\n',
+                                       '# COMMAND LINE\n']
+                job1_results.extend(command_line_header)
+                job1_results.append(cmdline1)
+                job2_results.extend(command_line_header)
+                job2_results.append(cmdline2)
+
+        if 'time' in args.diff_filter:
+            time1 = '%.2f s\n' % job1_data['time']
+            time2 = '%.2f s\n' % job2_data['time']
+
+            if str(time1) != str(time2):
+                total_time_header = ['\n',
+                                     '# TOTAL TIME\n']
+                job1_results.extend(total_time_header)
+                job1_results.append(time1)
+                job2_results.extend(total_time_header)
+                job2_results.append(time2)
+
+        if 'variants' in args.diff_filter:
+            variants1 = self._get_variants(job1_dir)
+            variants2 = self._get_variants(job2_dir)
+
+            if str(variants1) != str(variants2):
+                variants_header = ['\n',
+                                   '# VARIANTS\n']
+                job1_results.extend(variants_header)
+                job1_results.extend(variants1)
+                job2_results.extend(variants_header)
+                job2_results.extend(variants2)
+
+        if 'results' in args.diff_filter:
+            results1 = []
+            for test in job1_data['tests']:
+                test_result = '%s: %s\n' % (str(test['url']),
+                                            str(test['status']))
+                results1.append(test_result)
+            results2 = []
+            for test in job2_data['tests']:
+                test_result = '%s: %s\n' % (str(test['url']),
+                                            str(test['status']))
+                results2.append(test_result)
+
+            if str(results1) != str(results2):
+                test_results_header = ['\n',
+                                       '# TEST RESULTS\n']
+                job1_results.extend(test_results_header)
+                job1_results.extend(results1)
+                job2_results.extend(test_results_header)
+                job2_results.extend(results2)
+
+        if 'config' in args.diff_filter:
+            config1 = self._get_config(job1_dir)
+            config2 = self._get_config(job2_dir)
+
+            if str(config1) != str(config2):
+                config_header = ['\n',
+                                 '# SETTINGS\n']
+                job1_results.extend(config_header)
+                job1_results.extend(config1)
+                job2_results.extend(config_header)
+                job2_results.extend(config2)
+
+        if 'sysinfo' in args.diff_filter:
+            sysinfo_pre1 = self._get_sysinfo(job1_dir, 'pre')
+            sysinfo_pre2 = self._get_sysinfo(job2_dir, 'pre')
+
+            if str(sysinfo_pre1) != str(sysinfo_pre2):
+                sysinfo_header_pre = ['\n',
+                                      '# SYSINFO PRE\n']
+                job1_results.extend(sysinfo_header_pre)
+                job1_results.extend(sysinfo_pre1)
+                job2_results.extend(sysinfo_header_pre)
+                job2_results.extend(sysinfo_pre2)
+
+            sysinfo_post1 = self._get_sysinfo(job1_dir, 'post')
+            sysinfo_post2 = self._get_sysinfo(job2_dir, 'post')
+
+            if str(sysinfo_post1) != str(sysinfo_post2):
+                sysinfo_header_post = ['\n',
+                                       '# SYSINFO POST\n']
+                job1_results.extend(sysinfo_header_post)
+                job1_results.extend(sysinfo_post1)
+                job2_results.extend(sysinfo_header_post)
+                job2_results.extend(sysinfo_post2)
+
+        if getattr(args, 'create_reports', False):
+            self.std_diff_output = False
+            prefix = 'avocado_diff_%s_' % job1_id[:7]
+            tmp_file1 = tempfile.NamedTemporaryFile(prefix=prefix,
+                                                    suffix='.txt',
+                                                    delete=False)
+            tmp_file1.writelines(job1_results)
+            tmp_file1.close()
+
+            prefix = 'avocado_diff_%s_' % job2_id[:7]
+            tmp_file2 = tempfile.NamedTemporaryFile(prefix=prefix,
+                                                    suffix='.txt',
+                                                    delete=False)
+            tmp_file2.writelines(job2_results)
+            tmp_file2.close()
+
+            LOG.info('%s %s', tmp_file1.name, tmp_file2.name)
+
+        if (getattr(args, 'open_browser', False) and
+                getattr(args, 'html', None) is None):
+
+            prefix = 'avocado_diff_%s_%s_' % (job1_id[:7], job2_id[:7])
+            tmp_file = tempfile.NamedTemporaryFile(prefix=prefix,
+                                                   suffix='.html',
+                                                   delete=False)
+
+            setattr(args, 'html', tmp_file.name)
+
+        if getattr(args, 'html', None) is not None:
+            self.std_diff_output = False
+            try:
+                html_diff = HtmlDiff()
+                html_diff._legend = """
+                    <table class="diff" summary="Legends">
+                    <tr> <td> <table border="" summary="Colors">
+                    <tr><th> Colors </th> </tr>
+                    <tr><td class="diff_add">&nbsp;Added&nbsp;</td></tr>
+                    <tr><td class="diff_chg">Changed</td> </tr>
+                    <tr><td class="diff_sub">Deleted</td> </tr>
+                    </table></td>
+                    <td> <table border="" summary="Links">
+                    <tr><th colspan="2"> Links </th> </tr>
+                    <tr><td>(f)irst change</td> </tr>
+                    <tr><td>(n)ext change</td> </tr>
+                    <tr><td>(t)op</td> </tr>
+                    </table></td> </tr>
+                    </table>"""
+
+                job_diff_html = html_diff.make_file((_.decode("utf-8")
+                                                     for _ in job1_results),
+                                                    (_.decode("utf-8")
+                                                     for _ in job2_results),
+                                                    fromdesc=job1_id,
+                                                    todesc=job2_id)
+
+                with open(args.html, 'w') as html_file:
+                    html_file.writelines(job_diff_html.encode("utf-8"))
+
+                LOG.info(args.html)
+
+            except IOError as exception:
+                LOG.error(exception)
+                sys.exit(exit_codes.AVOCADO_FAIL)
+
+        if getattr(args, 'open_browser', False):
+            setsid = getattr(os, 'setsid', None)
+            if not setsid:
+                setsid = getattr(os, 'setpgrp', None)
+            with open(os.devnull, "r+") as inout:
+                cmd = ['xdg-open', args.html]
+                subprocess.Popen(cmd, close_fds=True, stdin=inout,
+                                 stdout=inout, stderr=inout,
+                                 preexec_fn=setsid)
+
+        if self.std_diff_output:
+            if self.term.enabled:
+                for line in self._cdiff(unified_diff(job1_results,
+                                                     job2_results,
+                                                     fromfile=job1_id,
+                                                     tofile=job2_id)):
+                    LOG.debug(line.strip())
+            else:
+                for line in unified_diff(job1_results,
+                                         job2_results,
+                                         fromfile=job1_id,
+                                         tofile=job2_id):
+                    LOG.debug(line.strip())
+
+    @staticmethod
+    def _validate_filters(string):
+        input_filter = set(string.split(','))
+        include_options = ["cmdline",
+                           "time",
+                           "variants",
+                           "results",
+                           "config",
+                           "sysinfo"]
+        exclude_options = ["nocmdline",
+                           "notime",
+                           "novariants",
+                           "noresults",
+                           "noconfig",
+                           "nosysinfo"]
+        invalid = input_filter.difference(include_options +
+                                          exclude_options + ["all"])
+        if invalid:
+            msg = "Invalid option(s) '%s'" % ','.join(invalid)
+            raise argparse.ArgumentTypeError(msg)
+        if input_filter.intersection(exclude_options):
+            output_filter = [_ for _ in include_options
+                             if ("no" + _) not in input_filter]
+        elif "all" in input_filter:
+            output_filter = include_options
+        else:
+            output_filter = input_filter
+
+        return output_filter
+
+    @staticmethod
+    def _get_job_data(jobdir):
+        results_json = os.path.join(jobdir, 'results.json')
+        with open(results_json, 'r') as json_file:
+            data = json.load(json_file)
+
+        return data
+
+    @staticmethod
+    def _setup_job(job_id):
+        if os.path.isdir(job_id):
+            resultsdir = os.path.expanduser(job_id)
+            job_id = ''
+        elif os.path.isfile(job_id):
+            resultsdir = os.path.dirname(os.path.expanduser(job_id))
+            job_id = ''
+        else:
+            logs_dir = settings.get_value('datadir.paths', 'logs_dir',
+                                          default=None)
+            logdir = os.path.expanduser(logs_dir)
+            resultsdir = replay.get_resultsdir(logdir, job_id)
+
+        if resultsdir is None:
+            LOG.error("Can't find job results directory for '%s' in '%s'",
+                      job_id, logdir)
+            sys.exit(exit_codes.AVOCADO_FAIL)
+
+        sourcejob = replay.get_id(os.path.join(resultsdir, 'id'), job_id)
+        if sourcejob is None:
+            LOG.error("Can't find matching job id '%s' in '%s' directory.",
+                      job_id, resultsdir)
+            sys.exit(exit_codes.AVOCADO_FAIL)
+
+        return resultsdir, sourcejob
+
+    @staticmethod
+    def _get_command_line(resultsdir):
+        command_line = replay.retrieve_cmdline(resultsdir)
+        if command_line is not None:
+            return '%s\n' % ' '.join(command_line)
+
+        return 'Not found\n'
+
+    @staticmethod
+    def _get_variants(resultsdir):
+        results = []
+        mux = replay.retrieve_mux(resultsdir)
+        if mux is not None:
+            env = set()
+            for (index, tpl) in enumerate(mux.variants):
+                paths = ', '.join([x.path for x in tpl])
+                results.append('Variant %s: %s\n' % (index + 1, paths))
+                for node in tpl:
+                    for key, value in node.environment.iteritems():
+                        origin = node.environment_origin[key].path
+                        env.add(("%s:%s" % (origin, key), str(value)))
+                if not env:
+                    continue
+                fmt = '    %%-%ds => %%s\n' % max([len(_[0]) for _ in env])
+                for record in sorted(env):
+                    results.append(fmt % record)
+        else:
+            results.append('Not found\n')
+
+        return results
+
+    @staticmethod
+    def _get_config(resultsdir):
+        config_file = os.path.join(resultsdir, 'replay', 'config')
+        try:
+            with open(config_file, 'r') as conf:
+                return conf.readlines()
+        except IOError:
+            return ['Not found\n']
+
+    @staticmethod
+    def _get_sysinfo(resultsdir, pre_post):
+        sysinfo_dir = os.path.join(resultsdir, 'sysinfo', pre_post)
+        sysinfo = []
+        for path, _, files in os.walk(sysinfo_dir):
+            for name in sorted(files):
+                name_header = ['\n', '** %s **\n' % name]
+                sysinfo.extend(name_header)
+                with open(os.path.join(path, name), 'r') as sysinfo_file:
+                    sysinfo.extend(sysinfo_file.readlines())
+
+        if sysinfo:
+            del sysinfo[0]
+
+        return sysinfo
+
+    def _cdiff(self, diff):
+        for line in diff:
+            if line.startswith('+'):
+                yield self.term.COLOR_GREEN + line
+            elif line.startswith('-'):
+                yield self.term.COLOR_RED + line
+            elif line.startswith('@'):
+                yield self.term.COLOR_BLUE + line
+            else:
+                yield line

docs/source/Diff.rst

+.. _job_diff_:
+
+========
+Job Diff
+========
+
+Avocado Diff plugin allows users to easily compare several aspects of
+two given jobs. The basic usage is::
+
+    $ avocado diff 7025aaba 384b949c
+    --- 7025aaba9c2ab8b4bba2e33b64db3824810bb5df
+    +++ 384b949c991b8ab324ce67c9d9ba761fd07672ff
+    @@ -1,15 +1,15 @@
+     
+     COMMAND LINE
+    -/usr/bin/avocado run sleeptest.py
+    +/usr/bin/avocado run passtest.py
+     
+     TOTAL TIME
+    -1.00 s
+    +0.00 s
+     
+     TEST RESULTS
+    -1-sleeptest.py:SleepTest.test: PASS
+    +1-passtest.py:PassTest.test: PASS
+     
+     ...
+
+Avocado Diff can compare and create an unified diff of:
+
+- Command line.
+- Job time.
+- Variants and parameters.
+- Tests results.
+- Configuration.
+- Sysinfo pre and post.
+
+Only sections with different content will be included in the results. You
+can also enable/disable those sections with ``--diff-filter``. Please see
+``avocado diff --help`` for more information.
+
+Jobs can be identified by the Job ID, by the results directory or by the
+key ``latest``. Example::
+
+    $ avocado diff ~/avocado/job-results/job-2016-08-03T15.56-4b3cb5b/ latest
+    --- 4b3cb5bbbb2435c91c7b557eebc09997d4a0f544
+    +++ 57e5bbb3991718b216d787848171b446f60b3262
+    @@ -1,9 +1,9 @@
+
+     COMMAND LINE
+    -/usr/bin/avocado run perfmon.py
+    +/usr/bin/avocado run passtest.py
+
+     TOTAL TIME
+    -11.91 s
+    +0.00 s
+
+     TEST RESULTS
+    -1-test.py:Perfmon.test: FAIL
+    +1-examples/tests/passtest.py:PassTest.test: PASS
+
+
+
+Along with the unified diff, you can also generate the html (option ``--html``)
+diff file and, optionally, open it on your preferred browser (option
+``--open-browser``)::
+
+
+    $ avocado diff 7025aaba 384b949c --html /tmp/myjobdiff.html
+    /tmp/myjobdiff.html
+
+If the option ``--open-browser`` is used without the ``--html``, we will
+create a temporary html file.
+
+For those wiling to use a custom diff tool instead of the Avocado Diff tool,
+we offer the option ``--create-reports``, so we create two temporary files
+with the relevant content. The file names are printed and user can copy/paste
+to the custom diff tool command line::
+
+    $ avocado diff 7025aaba 384b949c --create-reports
+    /var/tmp/avocado_diff_7025aab_zQJjJh.txt /var/tmp/avocado_diff_384b949_AcWq02.txt
+
+    $ diff -u /var/tmp/avocado_diff_7025aab_zQJjJh.txt /var/tmp/avocado_diff_384b949_AcWq02.txt
+    --- /var/tmp/avocado_diff_7025aab_zQJjJh.txt    2016-08-10 21:48:43.547776715 +0200
+    +++ /var/tmp/avocado_diff_384b949_AcWq02.txt    2016-08-10 21:48:43.547776715 +0200
+    @@ -1,250 +1,19 @@
+
+     COMMAND LINE
+     ============
+    -/usr/bin/avocado run sleeptest.py
+    +/usr/bin/avocado run passtest.py
+
+     TOTAL TIME
+     ==========
+    -1.00 s
+    +0.00 s
+
+    ...

docs/source/index.rst

@@ -16,6 +16,7 @@ Contents:
    LoggingSystem
    MultiplexConfig
    Replay
+   Diff
    RunningTestsRemotely
    DebuggingWithGDB
    WrapProcess

selftests/functional/test_plugin_diff.py

+#!/usr/bin/env python
+
+import glob
+import os
+import sys
+import tempfile
+import shutil
+
+if sys.version_info[:2] == (2, 6):
+    import unittest2 as unittest
+else:
+    import unittest
+
+from avocado.core import exit_codes
+from avocado.utils import process
+from avocado.utils import script
+
+
+basedir = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..')
+basedir = os.path.abspath(basedir)
+
+
+class DiffTests(unittest.TestCase):
+
+    def setUp(self):
+        self.tmpdir = tempfile.mkdtemp(prefix='avocado_' + __name__)
+        test = script.make_script(os.path.join(self.tmpdir, 'test'), 'exit 0')
+        cmd_line = ('./scripts/avocado run %s '
+                    '--external-runner /bin/bash '
+                    '--job-results-dir %s --sysinfo=off --json -' %
+                    (test, self.tmpdir))
+        expected_rc = exit_codes.AVOCADO_ALL_OK
+        self.run_and_check(cmd_line, expected_rc)
+        self.jobdir = ''.join(glob.glob(os.path.join(self.tmpdir, 'job-*')))
+
+        self.tmpdir2 = tempfile.mkdtemp(prefix='avocado_' + __name__)
+        cmd_line = ('./scripts/avocado run %s '
+                    '--external-runner /bin/bash '
+                    '--job-results-dir %s --sysinfo=off --json -' %
+                    (test, self.tmpdir2))
+        expected_rc = exit_codes.AVOCADO_ALL_OK
+        self.run_and_check(cmd_line, expected_rc)
+        self.jobdir2 = ''.join(glob.glob(os.path.join(self.tmpdir2, 'job-*')))
+
+    def run_and_check(self, cmd_line, expected_rc):
+        os.chdir(basedir)
+        result = process.run(cmd_line, ignore_status=True)
+        self.assertEqual(result.exit_status, expected_rc,
+                         "Command %s did not return rc "
+                         "%d:\n%s" % (cmd_line, expected_rc, result))
+        return result
+
+    def test_diff(self):
+        cmd_line = ('./scripts/avocado diff %s %s' %
+                    (self.jobdir, self.jobdir2))
+        expected_rc = exit_codes.AVOCADO_ALL_OK
+        result = self.run_and_check(cmd_line, expected_rc)
+        msg = "# COMMAND LINE"
+        self.assertIn(msg, result.stdout)
+        msg = "-./scripts/avocado run"
+        self.assertIn(msg, result.stdout)
+        msg = "+./scripts/avocado run"
+        self.assertIn(msg, result.stdout)
+
+    def test_diff_nocmdline(self):
+        cmd_line = ('./scripts/avocado diff %s %s --diff-filter nocmdline' %
+                    (self.jobdir, self.jobdir2))
+        expected_rc = exit_codes.AVOCADO_ALL_OK
+        result = self.run_and_check(cmd_line, expected_rc)
+        msg = "# COMMAND LINE"
+        self.assertNotIn(msg, result.stdout)
+
+    def tearDown(self):
+        shutil.rmtree(self.tmpdir)
+        shutil.rmtree(self.tmpdir2)
+
+
+if __name__ == '__main__':
+    unittest.main()

setup.py

@@ -139,6 +139,7 @@ if __name__ == '__main__':
                   'run = avocado.plugins.run:Run',
                   'sysinfo = avocado.plugins.sysinfo:SysInfo',
                   'plugins = avocado.plugins.plugins:Plugins',
+                  'diff = avocado.plugins.diff:Diff',
                   ],
               'avocado.plugins.job.prepost': [
                   'jobscripts = avocado.plugins.jobscripts:JobScripts',