Merge remote-tracking branch 'apahim/ldoktor_jobdata_compat'

Signed-off-by: NCleber Rosa <crosa@redhat.com>

.travis.yml

@@ -39,7 +39,7 @@ script:
             echo
             echo
             git checkout $COMMIT || ERR=$(echo -e "$ERR\nUnable to checkout $(git log -1 --oneline $COMMIT)")
-            AVOCADO_LOG_DEBUG=yes AVOCADO_RESULTSDIR_CHECK=y SELF_CHECK_CONTINUOUS=y AVOCADO_CHECK_LEVEL=1 make check || ERR=$(echo -e "$ERR\nmake check of $(git log -1 --oneline) failed")
+            AVOCADO_LOG_DEBUG=yes AVOCADO_RESULTSDIR_CHECK=y SELF_CHECK_CONTINUOUS=y AVOCADO_CHECK_LEVEL=1 AVOCADO_PARALLEL_CHECK=y make check || ERR=$(echo -e "$ERR\nmake check of $(git log -1 --oneline) failed")
             make clean
         done
         if [ "$ERR" ]; then

avocado/core/jobdata.py

@@ -21,7 +21,9 @@ import glob
 import json
 import os
 import pickle
+import sys
 
+from . import jobdata_compat_36_to_52
 from . import varianter
 from .output import LOG_UI, LOG_JOB
 from .settings import settings
@@ -41,6 +43,27 @@ ARGS_FILENAME = 'args'
 CMDLINE_FILENAME = 'cmdline'
 
 
+def _find_class(module, name):
+    """
+    Look for a class including compatibility workarounds
+    """
+    try:
+        mod = __import__(module)
+        mod = sys.modules[module]
+        return getattr(mod, name)
+    except ImportError:
+        if module == "avocado.core.multiplexer":
+            mod = __import__("avocado.core.jobdata_compat_36_to_52",
+                             fromlist=[module])
+            return getattr(mod, name)
+        elif module == "avocado.plugins.yaml_to_mux":
+            mod = __import__("avocado_varianter_yaml_to_mux",
+                             fromlist=[module])
+            return getattr(mod, name)
+        else:
+            raise
+
+
 def record(args, logdir, mux, references=None, cmdline=None):
     """
     Records all required job information.
@@ -130,6 +153,26 @@ def retrieve_variants(resultsdir):
     """
     Retrieves the job Mux object from the results directory.
     """
+    def _apply_36_to_52_workarounds(variants):
+        """
+        The 36.x version of TreeNode did not contain `filters`. Let's
+        re-initialize it per each child.
+        """
+        def get_fingerprint_meth(fingerprint):
+            """
+            36.x's TreeNode used to actually be equivalent of MuxTreeNode,
+            let's adjust the fingerprint to also contain self.ctrl
+            """
+            def get():
+                return fingerprint
+            return get
+        for node in variants.variants.root.iter_children_preorder():
+            node.filters = [[], []]
+            node._environment = None
+            fingerprint = node.fingerprint()
+            node.fingerprint = get_fingerprint_meth("%s%s" % (fingerprint,
+                                                              node.ctrl))
+
     recorded_mux = _retrieve(resultsdir, VARIANTS_FILENAME + ".json")
     if recorded_mux:    # new json-based dump
         with open(recorded_mux, 'r') as mux_file:
@@ -142,7 +185,14 @@ def retrieve_variants(resultsdir):
     # old pickle-based dump
     # TODO: Remove when 36lts is discontinued
     with open(recorded_mux, 'r') as mux_file:
-        return pickle.load(mux_file)
+        unpickler = pickle.Unpickler(mux_file)
+        unpickler.find_class = _find_class
+        variants = unpickler.load()
+        if isinstance(variants, jobdata_compat_36_to_52.Mux):
+            LOG_UI.warn("Using outdated 36.x variants file.")
+            _apply_36_to_52_workarounds(variants)
+        state = varianter.dump_ivariants(variants.itertests)
+        return varianter.Varianter(state=state)
 
 
 def retrieve_args(resultsdir):
@@ -178,6 +228,18 @@ def retrieve_cmdline(resultsdir):
     """
     recorded_cmdline = _retrieve(resultsdir, CMDLINE_FILENAME)
     if recorded_cmdline is None:
+        # Attemp to restore cmdline from log
+        try:
+            with open(os.path.join(resultsdir, "job.log"), "r") as log:
+                import re
+                cmd = re.search(r"# \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3} "
+                                r"\w{17}\w\d{4} INFO | Command line: (.*)",
+                                log.read())
+                if cmd:
+                    import shlex
+                    return shlex.split(cmd.group(1))
+        except IOError:
+            pass
         return None
     with open(recorded_cmdline, 'r') as cmdline_file:
         return ast.literal_eval(cmdline_file.read())

avocado/core/jobdata_compat_36_to_52.py

+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See LICENSE for more details.
+#
+# Copyright: Red Hat Inc. 2017
+#
+# Authors: Lukas Doktor <ldoktor@redhat.com>
+
+"""
+Jobdata compatibility layer from 36 to 52
+"""
+
+import hashlib
+import itertools
+
+from . import varianter
+from . import mux
+
+
+class MuxTree(mux.MuxPlugin):
+
+    """
+    Excerpt of MuxTree object in order to make it compatible with 52
+    """
+    pools = []
+    filters = [None, None]
+
+    def __iter__(self):
+        """
+        Iterates through variants
+        """
+        pools = []
+        for pool in self.pools:
+            if isinstance(pool, list):
+                pools.append(itertools.chain(*pool))
+            else:
+                pools.append(pool)
+        pools = itertools.product(*pools)
+        while True:
+            # TODO: Implement 2nd level filteres here
+            # TODO: This part takes most of the time, optimize it
+            yield list(itertools.chain(*pools.next()))
+
+
+class AvocadoParams(varianter.AvocadoParams):
+
+    """
+    Excerpt of original AvocadoParams in order to make it compatible
+    to the 52 version of AvocadoParams
+    """
+
+    def __init__(self, leaves, test_id, tag, mux_path, default_params):
+        """
+        :param leaves: List of TreeNode leaves defining current variant
+        :param test_id: test id
+        :param tag: test tag
+        :param mux_path: list of entry points
+        :param default_params: dict of params used when no matches found
+        """
+        del tag
+        super(AvocadoParams, self).__init__(leaves, test_id, mux_path,
+                                            default_params)
+
+
+class Mux(object):
+
+    """
+    Excerpt of Mux object in order to emulate compatible object to 52
+    """
+    variants = []
+    _mux_path = []
+
+    @staticmethod
+    def is_parsed():
+        """
+        For jobdata purpose we only report True
+        """
+        return True
+
+    def get_number_of_tests(self, test_suite):
+        """
+        :return: overall number of tests * multiplex variants
+        """
+        # Currently number of tests is symetrical
+        if self.variants:
+            no_variants = sum(1 for _ in self.variants)
+            if no_variants > 1:
+                self._has_multiple_variants = True
+            return (len(test_suite) * no_variants)
+        else:
+            return len(test_suite)
+
+    def dump(self):
+        return varianter.dump_ivariants(self.itertests)
+
+    @staticmethod
+    def _get_variant_id(variant):
+        variant.sort(key=lambda x: x.path)
+        fingerprint = "-".join(_.fingerprint() for _ in variant)
+        return ("-".join(node.name for node in variant) + '-' +
+                hashlib.sha1(fingerprint).hexdigest()[:4])
+
+    def itertests(self):
+        """
+        Processes the template and yields test definition with proper params
+        """
+        if self.variants:  # Copy template and modify it's params
+            handled = False
+            for variant in self.variants:
+                handled |= True
+                yield {"variant": variant,
+                       "variant_id": self._get_variant_id(variant),
+                       "mux_path": self._mux_path}
+            if not handled:   # No variants, use template
+                yield {"variant": [],
+                       "variant_id": None,
+                       "mux_path": "/run"}
+        else:   # No variants, use template
+            yield {"variant": [],
+                   "variant_id": None,
+                   "mux_path": "/run"}

avocado/core/mux.py

@@ -181,6 +181,10 @@ class MuxPlugin(object):
         """
         if self.root is None:
             return
+
+        # TODO: Remove when 36lts is disconinued
+        if not hasattr(self, "variant_ids"):
+            self.variant_ids = self._get_variant_ids()
         for vid, variant in itertools.izip(self.variant_ids, self.variants):
             yield {"variant_id": vid,
                    "variant": variant,

avocado/core/tree.py

@@ -116,6 +116,15 @@ class TreeNodeEnvOnly(object):
                 nodes[path] = TreeNodeEnvOnly(path)
             self.environment.origin[key] = nodes[path]
 
+    def __eq__(self, other):
+        if self.name != other.name:
+            return False
+        if self.path != other.path:
+            return False
+        if self.environment != other.environment:
+            return False
+        return True
+
     def get_environment(self):
         return self.environment
 

avocado/core/varianter.py

@@ -348,6 +348,30 @@ def variant_to_str(variant, verbosity, out_args=None, debug=False):
     return out
 
 
+def dump_ivariants(ivariants):
+    """
+    Walks the iterable variants and dumps them into json-serializable object
+    """
+    def dump_tree_node(node):
+        """
+        Turns TreeNode-like object into tuple(path, env_representation)
+        """
+        return (str(node.path),
+                [(str(node.environment.origin[key].path), str(key), value)
+                 for key, value in node.environment.iteritems()])
+
+    variants = []
+    for variant in ivariants():
+        safe_variant = {}
+        safe_variant["mux_path"] = [str(pth)
+                                    for pth in variant.get("mux_path")]
+        safe_variant["variant_id"] = str(variant.get("variant_id"))
+        safe_variant["variant"] = [dump_tree_node(_)
+                                   for _ in variant.get("variant", [])]
+        variants.append(safe_variant)
+    return variants
+
+
 class FakeVariantDispatcher(object):
 
     """
@@ -370,6 +394,26 @@ class FakeVariantDispatcher(object):
         else:
             return []
 
+    def to_str(self, summary=0, variants=0, **kwargs):
+        if not self.variants:
+            return ""
+        out = []
+        for variant in self.variants:
+            paths = ', '.join([x.path for x in variant["variant"]])
+            out.append('\nVariant %s:    %s' % (variant["variant_id"],
+                                                paths))
+            env = set()
+            for node in variant["variant"]:
+                for key, value in node.environment.iteritems():
+                    origin = node.environment.origin[key].path
+                    env.add(("%s:%s" % (origin, key), str(value)))
+            if not env:
+                continue
+            fmt = '    %%-%ds => %%s' % max([len(_[0]) for _ in env])
+            for record in sorted(env):
+                out.append(fmt % record)
+        return "\n".join(out)
+
     def __iter__(self):
         return iter(self.variants)
 
@@ -519,28 +563,10 @@ class Varianter(object):
 
         :return: loadable Varianter representation
         """
-        def dump_tree_node(node):
-            """
-            Turns TreeNode-like object into tuple(path, env_representation)
-            """
-            return (str(node.path),
-                    [(str(node.environment.origin[key].path), str(key), value)
-                     for key, value in node.environment.iteritems()])
-
         if not self.is_parsed():
             raise NotImplementedError("Dumping Varianter state before "
                                       "multiplexation is not supported.")
-        variants = []
-        for variant in self.itertests():
-            safe_variant = {}
-            safe_variant["mux_path"] = [str(pth)
-                                        for pth in variant.get("mux_path")]
-            safe_variant["variant_id"] = str(variant.get("variant_id"))
-            safe_variant["variant"] = [dump_tree_node(_)
-                                       for _ in variant.get("variant", [])]
-            variants.append(safe_variant)
-
-        return variants
+        return dump_ivariants(self.itertests)
 
     def load(self, state):
         """

avocado/plugins/replay.py

@@ -223,7 +223,7 @@ class Replay(CLI):
                                 "given on the command line.",
                                 option.replace('_', '-'),
                                 option.replace('_', '-'))
-                else:
+                elif option in replay_args:
                     setattr(args, option, replay_args[option])
 
         # Keeping this for compatibility.

selftests/.data/results-36.0lts/id

+123456

selftests/.data/results-36.0lts/job.log

+2017-06-22 18:27:22,725 sysinfo          L0424 INFO | System log file not found (looked for ['/var/log/messages', '/var/log/syslog', '/var/log/system.log'])
+2017-06-22 18:27:22,725 job              L0364 INFO | Command line: /usr/local/bin/avocado run --external-runner /bin/echo -m examples/mux-0.yaml -- yes no
+2017-06-22 18:27:22,725 job              L0365 INFO | 

selftests/.data/results-36.0lts/replay/config

+[datadir.paths]
+base_dir = /usr/share/avocado
+test_dir = /usr/share/avocado/tests
+data_dir = /usr/share/avocado/data
+logs_dir = ~/avocado/job-results
+
+[sysinfo.collect]
+enabled = True
+installed_packages = False
+profiler = False
+
+[sysinfo.collectibles]
+commands = /etc/avocado/sysinfo/commands
+files = /etc/avocado/sysinfo/files
+profilers = /etc/avocado/sysinfo/profilers
+
+[runner.output]
+colored = True
+utf8 = 
+
+[runner.behavior]
+keep_tmp_files = False
+
+[remoter.behavior]
+reject_unknown_hosts = False
+disable_known_hosts = False
+
+[job.output]
+loglevel = debug
+
+[restclient.connection]
+hostname = localhost
+port = 9405
+username = 
+password = 
+
+[plugins]
+skip_broken_plugin_notification = []
+loaders = ['file', '@DEFAULT']
+
+[plugins.vtjoblock]
+dir = /tmp
+
+[gdb.paths]
+gdb = /usr/bin/gdb
+gdbserver = /usr/bin/gdbserver
+
+[plugins.jobscripts]
+pre = /etc/avocado/scripts/job/pre.d/
+post = /etc/avocado/scripts/job/post.d/
+warn_non_existing_dir = False
+warn_non_zero_status = True
+
+[avocado.selftest]
+jobdata = yes

selftests/.data/results-36.0lts/replay/pwd

+/home/medic/Work/Projekty/avocado/avocado
\ No newline at end of file

selftests/.data/results-36.0lts/replay/urls

+['yes', 'no']
\ No newline at end of file

selftests/.data/results-36.4/id

+123456

selftests/.data/results-36.4/job.log

+2017-06-22 18:27:22,725 sysinfo          L0424 INFO | System log file not found (looked for ['/var/log/messages', '/var/log/syslog', '/var/log/system.log'])
+2017-06-22 18:27:22,725 job              L0364 INFO | Command line: /usr/local/bin/avocado run --external-runner /bin/echo -m examples/mux-0.yaml -- yes no
+2017-06-22 18:27:22,725 job              L0365 INFO | 

selftests/.data/results-36.4/replay/config

+[datadir.paths]
+base_dir = /usr/share/avocado
+test_dir = /usr/share/avocado/tests
+data_dir = /usr/share/avocado/data
+logs_dir = ~/avocado/job-results
+
+[sysinfo.collect]
+enabled = True
+installed_packages = False
+profiler = False
+
+[sysinfo.collectibles]
+commands = /etc/avocado/sysinfo/commands
+files = /etc/avocado/sysinfo/files
+profilers = /etc/avocado/sysinfo/profilers
+
+[runner.output]
+colored = True
+utf8 = 
+
+[runner.behavior]
+keep_tmp_files = False
+
+[remoter.behavior]
+reject_unknown_hosts = False
+disable_known_hosts = False
+
+[job.output]
+loglevel = debug
+
+[restclient.connection]
+hostname = localhost
+port = 9405
+username = 
+password = 
+
+[plugins]
+skip_broken_plugin_notification = []
+loaders = ['file', '@DEFAULT']
+
+[plugins.vtjoblock]
+dir = /tmp
+
+[gdb.paths]
+gdb = /usr/bin/gdb
+gdbserver = /usr/bin/gdbserver
+
+[plugins.jobscripts]
+pre = /etc/avocado/scripts/job/pre.d/
+post = /etc/avocado/scripts/job/post.d/
+warn_non_existing_dir = False
+warn_non_zero_status = True
+
+[avocado.selftest]
+jobdata = yes

selftests/.data/results-36.4/replay/pwd

+/home/medic/Work/Projekty/avocado/avocado
\ No newline at end of file

selftests/.data/results-36.4/replay/urls

+['yes', 'no']
\ No newline at end of file

selftests/.data/results-37.0/id

+123456

selftests/.data/results-37.0/job.log

+2017-06-22 18:27:22,725 sysinfo          L0424 INFO | System log file not found (looked for ['/var/log/messages', '/var/log/syslog', '/var/log/system.log'])
+2017-06-22 18:27:22,725 job              L0364 INFO | Command line: /usr/local/bin/avocado run --external-runner /bin/echo -m examples/mux-0.yaml -- yes no
+2017-06-22 18:27:22,725 job              L0365 INFO | 

selftests/.data/results-37.0/replay/config

+[datadir.paths]
+base_dir = /usr/share/avocado
+test_dir = /usr/share/avocado/tests
+data_dir = /usr/share/avocado/data
+logs_dir = ~/avocado/job-results
+
+[sysinfo.collect]
+enabled = True
+installed_packages = False
+profiler = False
+
+[sysinfo.collectibles]
+commands = /etc/avocado/sysinfo/commands
+files = /etc/avocado/sysinfo/files
+profilers = /etc/avocado/sysinfo/profilers
+
+[runner.output]
+colored = True
+utf8 = 
+
+[runner.behavior]
+keep_tmp_files = False
+
+[remoter.behavior]
+reject_unknown_hosts = False
+disable_known_hosts = False
+
+[job.output]
+loglevel = debug
+
+[restclient.connection]
+hostname = localhost
+port = 9405
+username = 
+password = 
+
+[plugins]
+skip_broken_plugin_notification = []
+loaders = ['file', '@DEFAULT']
+
+[plugins.vtjoblock]
+dir = /tmp
+
+[gdb.paths]
+gdb = /usr/bin/gdb
+gdbserver = /usr/bin/gdbserver
+
+[plugins.jobscripts]
+pre = /etc/avocado/scripts/job/pre.d/
+post = /etc/avocado/scripts/job/post.d/
+warn_non_existing_dir = False
+warn_non_zero_status = True
+
+[avocado.selftest]
+jobdata = yes

selftests/.data/results-37.0/replay/pwd

+/home/medic/Work/Projekty/avocado/avocado
\ No newline at end of file

selftests/.data/results-37.0/replay/urls

+['yes', 'no']
\ No newline at end of file

selftests/.data/results-38.0/id

+123456

selftests/.data/results-38.0/job.log

+2017-06-22 18:27:22,725 sysinfo          L0424 INFO | System log file not found (looked for ['/var/log/messages', '/var/log/syslog', '/var/log/system.log'])
+2017-06-22 18:27:22,725 job              L0364 INFO | Command line: /usr/local/bin/avocado run --external-runner /bin/echo -m examples/mux-0.yaml -- yes no
+2017-06-22 18:27:22,725 job              L0365 INFO | 

selftests/.data/results-38.0/replay/config

+[datadir.paths]
+base_dir = /usr/share/avocado
+test_dir = /usr/share/avocado/tests
+data_dir = /usr/share/avocado/data
+logs_dir = ~/avocado/job-results
+
+[sysinfo.collect]
+enabled = True
+installed_packages = False
+profiler = False
+
+[sysinfo.collectibles]
+commands = /etc/avocado/sysinfo/commands
+files = /etc/avocado/sysinfo/files
+profilers = /etc/avocado/sysinfo/profilers
+
+[runner.output]
+colored = True
+utf8 = 
+
+[runner.behavior]
+keep_tmp_files = False
+
+[remoter.behavior]
+reject_unknown_hosts = False
+disable_known_hosts = False
+
+[job.output]
+loglevel = debug
+
+[restclient.connection]
+hostname = localhost
+port = 9405
+username = 
+password = 
+
+[plugins]
+skip_broken_plugin_notification = []
+loaders = ['file', '@DEFAULT']
+
+[plugins.vtjoblock]
+dir = /tmp
+
+[gdb.paths]
+gdb = /usr/bin/gdb
+gdbserver = /usr/bin/gdbserver
+
+[plugins.jobscripts]
+pre = /etc/avocado/scripts/job/pre.d/
+post = /etc/avocado/scripts/job/post.d/
+warn_non_existing_dir = False
+warn_non_zero_status = True
+
+[avocado.selftest]
+jobdata = yes

selftests/.data/results-38.0/replay/pwd

+/home/medic/Work/Projekty/avocado/avocado
\ No newline at end of file

selftests/.data/results-38.0/replay/urls

+['yes', 'no']
\ No newline at end of file

selftests/.data/results-39.0/id

+123456

selftests/.data/results-39.0/job.log

+2017-06-22 18:27:22,725 sysinfo          L0424 INFO | System log file not found (looked for ['/var/log/messages', '/var/log/syslog', '/var/log/system.log'])
+2017-06-22 18:27:22,725 job              L0364 INFO | Command line: /usr/local/bin/avocado run --external-runner /bin/echo -m examples/mux-0.yaml -- yes no
+2017-06-22 18:27:22,725 job              L0365 INFO | 

selftests/.data/results-39.0/replay/config

+[datadir.paths]
+base_dir = /usr/share/avocado
+test_dir = /usr/share/avocado/tests
+data_dir = /usr/share/avocado/data
+logs_dir = ~/avocado/job-results
+
+[sysinfo.collect]
+enabled = True
+installed_packages = False
+profiler = False
+
+[sysinfo.collectibles]
+commands = /etc/avocado/sysinfo/commands
+files = /etc/avocado/sysinfo/files
+profilers = /etc/avocado/sysinfo/profilers
+
+[runner.output]
+colored = True
+utf8 = 
+
+[runner.behavior]
+keep_tmp_files = False
+
+[remoter.behavior]
+reject_unknown_hosts = False
+disable_known_hosts = False
+
+[job.output]
+loglevel = debug
+
+[restclient.connection]
+hostname = localhost
+port = 9405
+username = 
+password = 
+
+[plugins]
+skip_broken_plugin_notification = []
+loaders = ['file', '@DEFAULT']
+
+[plugins.vtjoblock]
+dir = /tmp
+
+[gdb.paths]
+gdb = /usr/bin/gdb
+gdbserver = /usr/bin/gdbserver
+
+[plugins.jobscripts]
+pre = /etc/avocado/scripts/job/pre.d/
+post = /etc/avocado/scripts/job/post.d/
+warn_non_existing_dir = False
+warn_non_zero_status = True
+
+[avocado.selftest]
+jobdata = yes

selftests/.data/results-39.0/replay/pwd

+/home/medic/Work/Projekty/avocado/avocado
\ No newline at end of file

selftests/.data/results-39.0/replay/urls

+['yes', 'no']
\ No newline at end of file

selftests/.data/results-40.0/id

+123456

selftests/.data/results-40.0/replay/cmdline

+['/usr/local/bin/avocado', 'run', '--external-runner', '/bin/echo', '-m', 'examples/mux-0.yaml', '--', 'yes', 'no']
\ No newline at end of file

selftests/.data/results-40.0/replay/config

+[datadir.paths]
+base_dir = /usr/share/avocado
+test_dir = /usr/share/avocado/tests
+data_dir = /usr/share/avocado/data
+logs_dir = ~/avocado/job-results
+
+[sysinfo.collect]
+enabled = True
+installed_packages = False
+profiler = False
+
+[sysinfo.collectibles]
+commands = /etc/avocado/sysinfo/commands
+files = /etc/avocado/sysinfo/files
+profilers = /etc/avocado/sysinfo/profilers
+
+[runner.output]
+colored = True
+utf8 = 
+
+[runner.behavior]
+keep_tmp_files = False
+
+[remoter.behavior]
+reject_unknown_hosts = False
+disable_known_hosts = False
+
+[job.output]
+loglevel = debug
+
+[restclient.connection]
+hostname = localhost
+port = 9405
+username = 
+password = 
+
+[plugins]
+skip_broken_plugin_notification = []
+loaders = ['file', '@DEFAULT']
+
+[plugins.vtjoblock]
+dir = /tmp
+
+[gdb.paths]
+gdb = /usr/bin/gdb
+gdbserver = /usr/bin/gdbserver
+
+[plugins.jobscripts]
+pre = /etc/avocado/scripts/job/pre.d/
+post = /etc/avocado/scripts/job/post.d/
+warn_non_existing_dir = False
+warn_non_zero_status = True
+
+[avocado.selftest]
+jobdata = yes

selftests/.data/results-40.0/replay/pwd

+/home/medic/Work/Projekty/avocado/avocado
\ No newline at end of file

selftests/.data/results-40.0/replay/urls

+['yes', 'no']
\ No newline at end of file

selftests/.data/results-41.0/id

+123456

selftests/.data/results-41.0/jobdata/cmdline

+['/usr/local/bin/avocado', 'run', '--external-runner', '/bin/echo', '-m', 'examples/mux-0.yaml', '--', 'yes', 'no']
\ No newline at end of file

selftests/.data/results-41.0/jobdata/config

+[datadir.paths]
+base_dir = /usr/share/avocado
+test_dir = /usr/share/avocado/tests
+data_dir = /usr/share/avocado/data
+logs_dir = ~/avocado/job-results
+
+[sysinfo.collect]
+enabled = True
+installed_packages = False
+profiler = False
+locale = C
+
+[sysinfo.collectibles]
+commands = /etc/avocado/sysinfo/commands
+files = /etc/avocado/sysinfo/files
+profilers = /etc/avocado/sysinfo/profilers
+
+[runner.output]
+colored = True
+utf8 = 
+
+[runner.behavior]
+keep_tmp_files = False
+
+[remoter.behavior]
+reject_unknown_hosts = False
+disable_known_hosts = False
+
+[job.output]
+loglevel = debug
+
+[restclient.connection]
+hostname = localhost
+port = 9405
+username = 
+password = 
+
+[plugins]
+skip_broken_plugin_notification = []
+loaders = ['file', '@DEFAULT']
+
+[plugins.vtjoblock]
+dir = /tmp
+
+[gdb.paths]
+gdb = /usr/bin/gdb
+gdbserver = /usr/bin/gdbserver
+
+[plugins.jobscripts]
+pre = /etc/avocado/scripts/job/pre.d/
+post = /etc/avocado/scripts/job/post.d/
+warn_non_existing_dir = False
+warn_non_zero_status = True
+
+[avocado.selftest]
+jobdata = yes

selftests/.data/results-41.0/jobdata/pwd

+/home/medic/Work/Projekty/avocado/avocado
\ No newline at end of file

selftests/.data/results-41.0/jobdata/urls

+['yes', 'no']
\ No newline at end of file

selftests/.data/results-51.0/id

+123456

selftests/.data/results-51.0/jobdata/args.json

+{"replay_teststatus": null, "external_runner_chdir": null, "unique_job_id": null, "show": ["app", "debug"], "filter_out": [], "vm_username": "medic", "docker_cmd": "docker", "execution_order": "variants-per-test", "remote_password": null, "mux_filter_only": [], "html_job_result": "on", "html_output": null, "tap": null, "replay_ignore": [], "archive": false, "xunit_output": null, "show_job_log": false, "external_runner_testdir": null, "output_check": "on", "replay_jobid": null, "remote_username": "medic", "remote_port": 22, "filter_only": [], "sysinfo": "on", "remote_key_file": null, "remote_hostname": null, "wrapper": [], "loaders": ["external:/bin/echo"], "job_timeout": 0, "mux_inject": [], "logdir": null, "gdb_prerun_commands": [], "vm_domain": null, "store_logging_stream": [], "external_runner": "/bin/echo", "vm_hostname": null, "vm_key_file": null, "config": null, "journal": false, "json_job_result": "on", "vm_password": null, "gdb_coredump": "off", "vm_timeout": 120, "remote_timeout": 60, "xunit_job_result": "on", "subcommand": "run", "mux_filter_out": [], "keep_tmp": "off", "docker_no_cleanup": false, "mux_path": null, "avocado_variants": null, "open_browser": false, "json_output": null, "vm_port": 22, "docker_options": "", "reference": ["yes", "no"], "tap_job_result": "on", "resultsdb_logs": null, "env_keep": null, "resultsdb_api": null, "dry_run": false, "vm_cleanup": false, "mux_yaml": ["examples/mux-0.yaml"], "vm_hypervisor_uri": "qemu:///system", "replay_resume": false, "output_check_record": "none", "filter_by_tags": null, "multiplex": null, "filter_by_tags_include_empty": false, "ignore_missing_references": null, "default_avocado_params": null, "docker": null, "failfast": null, "gdb_run_bin": []}
\ No newline at end of file

selftests/.data/results-51.0/jobdata/cmdline

+['/usr/local/bin/avocado', 'run', '--external-runner', '/bin/echo', '-m', 'examples/mux-0.yaml', '--', 'yes', 'no']
\ No newline at end of file

selftests/.data/results-51.0/jobdata/config

+[datadir.paths]
+base_dir = /var/lib/avocado
+test_dir = /usr/share/avocado/tests
+data_dir = /var/lib/avocado/data
+logs_dir = ~/avocado/job-results
+
+[sysinfo.collect]
+enabled = True
+commands_timeout = -1
+installed_packages = False
+profiler = False
+locale = C
+
+[sysinfo.collectibles]
+commands = /etc/avocado/sysinfo/commands
+files = /etc/avocado/sysinfo/files
+profilers = /etc/avocado/sysinfo/profilers
+
+[runner.output]
+colored = True
+utf8 = 
+
+[remoter.behavior]
+reject_unknown_hosts = False
+disable_known_hosts = False
+
+[job.output]
+loglevel = debug
+
+[restclient.connection]
+hostname = localhost
+port = 9405
+username = 
+password = 
+
+[plugins]
+disable = []
+skip_broken_plugin_notification = []
+loaders = ['file', '@DEFAULT']
+
+[plugins.vtjoblock]
+dir = /tmp
+
+[gdb.paths]
+gdb = /usr/bin/gdb
+gdbserver = /usr/bin/gdbserver
+
+[plugins.jobscripts]
+pre = /etc/avocado/scripts/job/pre.d/
+post = /etc/avocado/scripts/job/post.d/
+warn_non_existing_dir = False
+warn_non_zero_status = True
+
+[plugins.resultsdb]
+
+[avocado.selftest]
+jobdata = yes

selftests/.data/results-51.0/jobdata/pwd

+/home/medic/Work/Projekty/avocado/avocado
\ No newline at end of file

selftests/.data/results-51.0/jobdata/test_references

+['yes', 'no']
\ No newline at end of file

selftests/.data/results-51.0/jobdata/urls

+test_references
\ No newline at end of file

selftests/.data/results-51.0/jobdata/variants.json

+[{"mux_path": ["/run/*"], "variant": [["/run/variant1", [["/run/variant1", "foo", "bar"]]]], "variant_id": "variant1-6ec4"}, {"mux_path": ["/run/*"], "variant": [["/run/variant2", [["/run/variant2", "foo", "baz"]]]], "variant_id": "variant2-a6fe"}]
\ No newline at end of file

selftests/unit/test_jobdata.py

+import glob
+import os
+import unittest
+
+from avocado.core import jobdata
+
+
+BASEDIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..')
+BASEDIR = os.path.abspath(BASEDIR)
+
+
+class JobdataTest(unittest.TestCase):
+
+    @staticmethod
+    def _check_results(pth):
+        msg = "Retrieved %s is not '%s' (%s)"
+        errs = []
+        # pwd
+        exp = "/home/medic/Work/Projekty/avocado/avocado"
+        act = jobdata.retrieve_pwd(pth)
+        if act != exp:
+            errs.append("pwd: '%s' '%s'" % (exp, act))
+
+        # references
+        exp = ["yes", "no"]
+        act = jobdata.retrieve_references(pth)
+        if act != exp:
+            errs.append("references: '%s' '%s'" % (exp, act))
+
+        # variants
+        try:
+            variants = jobdata.retrieve_variants(pth)
+            act = variants.to_str(0, 99)
+        except Exception as details:
+            errs.append("variants: Unable to retrieve: %s" % details)
+        else:
+            exp = ("\nVariant variant1-6ec4:    /run/variant1\n"
+                   "    /run/variant1:foo => bar\n\n"
+                   "Variant variant2-a6fe:    /run/variant2\n"
+                   "    /run/variant2:foo => baz")
+            if not act or exp not in act:
+                errs.append("variants:\n%s\n\n%s" % (exp, act))
+
+        # args
+        try:
+            args = jobdata.retrieve_args(pth)
+        except Exception as details:
+            errs.append("args: Unable to retrieve: %s" % details)
+        else:
+            if isinstance(args, dict):
+                for scenario in [["loaders", [u"external:/bin/echo"]],
+                                 ["external_runner", u"/bin/echo"],
+                                 ["failfast", False, None],
+                                 ["ignore_missing_references", False, None],
+                                 ["execution_order", "variants-per-test",
+                                  None]]:
+                    act = args.get(scenario[0])
+                    for exp in scenario[1:]:
+                        if act == exp:
+                            break
+                    else:
+                        errs.append("args: Invalid value '%s' of key '%s' "
+                                    "%s" % (act, scenario[0],
+                                            scenario[1:]))
+            else:
+                errs.append("args: Invalid args: %s" % args)
+
+        # config
+        conf_path = jobdata.retrieve_config(pth)
+        if os.path.exists(conf_path):
+            exp = "[avocado.selftest]\njobdata = yes"
+            with open(conf_path, "r") as conf:
+                act = conf.read()
+                if exp not in act:
+                    errs.append("config: Expected string\n%s\n\nNot in:\n%s"
+                                % (exp, act))
+
+        else:
+            errs.append("config: Retrieved path '%s' does not exists"
+                        % conf_path)
+
+        # cmdline
+        act = jobdata.retrieve_cmdline(pth)
+        exp = ['/usr/local/bin/avocado', 'run', '--external-runner',
+               '/bin/echo', '-m', 'examples/mux-0.yaml', '--', 'yes', 'no']
+        if exp != act:
+            errs.append("cmdline: Invalid cmdline '%s' (%s)" % (act, exp))
+        return errs
+
+    def test_versions(self):
+        os.chdir(BASEDIR)
+        errs = []
+        for pth in sorted(glob.glob(os.path.join(BASEDIR, "selftests",
+                                                 ".data", "results-*"))):
+            res = self._check_results(pth)
+            if res:
+                name = os.path.basename(pth)
+                errs.append("%s\n%s\n\n  %s\n\n" % (name, "-" * len(name),
+                                                    "\n  ".join(res)))
+        self.assertFalse(errs, "Some results were not loaded properly:\n%s"
+                         % "\n * ".join(errs))
+
+
+if __name__ == "__main__":
+    unittest.main()