Merge branch 'ldoktor-jobdata-varianter2'

Signed-off-by: NAmador Pahim <apahim@redhat.com>

avocado/core/jobdata.py

@@ -18,9 +18,12 @@ Record/retrieve job information
 
 import ast
 import glob
+import json
 import os
 import pickle
 
+from . import varianter
+from .output import LOG_UI, LOG_JOB
 from .settings import settings
 from ..utils.path import init_dir
 
@@ -30,7 +33,9 @@ JOB_DATA_FALLBACK_DIR = 'replay'
 CONFIG_FILENAME = 'config'
 TEST_REFERENCES_FILENAME = 'test_references'
 TEST_REFERENCES_FILENAME_LEGACY = 'urls'
-VARIANTS_FILENAME = 'multiplex'
+VARIANTS_FILENAME = 'variants'
+# TODO: Remove when 36lts is discontinued
+VARIANTS_FILENAME_LEGACY = 'multiplex'
 PWD_FILENAME = 'pwd'
 ARGS_FILENAME = 'args'
 CMDLINE_FILENAME = 'cmdline'
@@ -40,14 +45,18 @@ def record(args, logdir, mux, references=None, cmdline=None):
     """
     Records all required job information.
     """
+    def json_bad_mux_obj(item):
+        for log in [LOG_UI, LOG_JOB]:
+            log.warning("jobdata.variants: Unable to serialize '%s'", item)
+        return str(item)
     base_dir = init_dir(logdir, JOB_DATA_DIR)
     path_cfg = os.path.join(base_dir, CONFIG_FILENAME)
     path_references = os.path.join(base_dir, TEST_REFERENCES_FILENAME)
     path_references_legacy = os.path.join(base_dir,
                                           TEST_REFERENCES_FILENAME_LEGACY)
-    path_mux = os.path.join(base_dir, VARIANTS_FILENAME)
+    path_mux = os.path.join(base_dir, VARIANTS_FILENAME + ".json")
     path_pwd = os.path.join(base_dir, PWD_FILENAME)
-    path_args = os.path.join(base_dir, ARGS_FILENAME)
+    path_args = os.path.join(base_dir, ARGS_FILENAME + ".json")
     path_cmdline = os.path.join(base_dir, CMDLINE_FILENAME)
 
     if references:
@@ -63,7 +72,7 @@ def record(args, logdir, mux, references=None, cmdline=None):
         os.fsync(config_file)
 
     with open(path_mux, 'w') as mux_file:
-        pickle.dump(mux, mux_file, pickle.HIGHEST_PROTOCOL)
+        json.dump(mux.dump(), mux_file, default=json_bad_mux_obj)
         mux_file.flush()
         os.fsync(mux_file)
 
@@ -73,7 +82,7 @@ def record(args, logdir, mux, references=None, cmdline=None):
         os.fsync(pwd_file)
 
     with open(path_args, 'w') as args_file:
-        pickle.dump(args.__dict__, args_file, pickle.HIGHEST_PROTOCOL)
+        json.dump(args.__dict__, args_file, default=lambda x: None)
         args_file.flush()
         os.fsync(args_file)
 
@@ -121,9 +130,17 @@ def retrieve_variants(resultsdir):
     """
     Retrieves the job Mux object from the results directory.
     """
+    recorded_mux = _retrieve(resultsdir, VARIANTS_FILENAME + ".json")
+    if recorded_mux:    # new json-based dump
+        with open(recorded_mux, 'r') as mux_file:
+            return varianter.Varianter(state=json.load(mux_file))
     recorded_mux = _retrieve(resultsdir, VARIANTS_FILENAME)
+    if recorded_mux is None:
+        recorded_mux = _retrieve(resultsdir, VARIANTS_FILENAME_LEGACY)
     if recorded_mux is None:
         return None
+    # old pickle-based dump
+    # TODO: Remove when 36lts is discontinued
     with open(recorded_mux, 'r') as mux_file:
         return pickle.load(mux_file)
 
@@ -132,9 +149,15 @@ def retrieve_args(resultsdir):
     """
     Retrieves the job args from the results directory.
     """
+    recorded_args = _retrieve(resultsdir, ARGS_FILENAME + ".json")
+    if recorded_args:
+        with open(recorded_args, 'r') as args_file:
+            return json.load(args_file)
     recorded_args = _retrieve(resultsdir, ARGS_FILENAME)
     if recorded_args is None:
         return None
+    # old pickle-based dump
+    # TODO: Remove when 36lts is discontinued
     with open(recorded_args, 'r') as args_file:
         return pickle.load(args_file)
 

avocado/core/tree.py

@@ -92,6 +92,37 @@ class TreeEnvironment(dict):
                          str(self.filter_out)))
 
 
+class TreeNodeEnvOnly(object):
+
+    """
+    Minimal TreeNode-like class providing interface for AvocadoParams
+    """
+    def __init__(self, path, environment=None):
+        """
+        :param path: Path of this node (must not end with '/')
+        :param environment: List of pair/key/value items
+        """
+        self.name = path.rsplit("/")[-1]
+        self.path = path
+        self.environment = TreeEnvironment()
+        if environment:
+            self.__load_environment(environment)
+
+    def __load_environment(self, environment):
+        nodes = {}
+        for path, key, value in environment:
+            self.environment[key] = value
+            if path not in nodes:
+                nodes[path] = TreeNodeEnvOnly(path)
+            self.environment.origin[key] = nodes[path]
+
+    def get_environment(self):
+        return self.environment
+
+    def get_path(self):
+        return self.path
+
+
 class TreeNode(object):
 
     """

avocado/core/varianter.py

@@ -26,7 +26,6 @@ from . import dispatcher
 from .output import LOG_JOB
 
 
-# TODO: Create multiplexer plugin and split these functions into multiple files
 class NoMatchError(KeyError):
     pass
 
@@ -310,24 +309,57 @@ class AvocadoParam(object):
                 yield (leaf.environment.origin[key].path, key, value)
 
 
+class FakeVariantDispatcher(object):
+
+    """
+    This object can act instead of VarianterDispatcher to report loaded
+    variants.
+    """
+
+    def __init__(self, state):
+        for variant in state:
+            variant["variant"] = [tree.TreeNodeEnvOnly(path, env)
+                                  for path, env in variant["variant"]]
+        self.variants = state
+
+    def map_method(self, method, *args, **kwargs):
+        """
+        Reports list containing one result of map_method on self
+        """
+        if hasattr(self, method):
+            return [getattr(self, method)(*args, **kwargs)]
+        else:
+            return []
+
+    def __iter__(self):
+        return iter(self.variants)
+
+    def __len__(self):
+        return sum(1 for _ in self)
+
+
 class Varianter(object):
 
     """
     This object takes care of producing test variants
     """
 
-    def __init__(self, debug=False):
+    def __init__(self, debug=False, state=None):
         """
         :param debug: Store whether this instance should debug the mux
+        :param state: Force-varianter state
         :note: people need to check whether mux uses debug and reflect that
                in order to provide the right results.
         """
         self.default_params = {}
         self._default_params = None
-        self.debug = debug
-        self.node_class = tree.TreeNode if not debug else tree.TreeNodeDebug
-        self._variant_plugins = dispatcher.VarianterDispatcher()
-        self._no_variants = None
+        if state is None:
+            self.debug = debug
+            self.node_class = tree.TreeNodeDebug if debug else tree.TreeNode
+            self._variant_plugins = dispatcher.VarianterDispatcher()
+            self._no_variants = None
+        else:
+            self.load(state)
 
     def parse(self, args):
         """
@@ -410,6 +442,72 @@ class Varianter(object):
         else:
             return len(test_suite)
 
+    def dump(self):
+        """
+        Dump the variants in loadable-state
+
+        This is lossy representation which takes all yielded variants and
+        replaces the list of nodes with TreeNodeEnvOnly representations::
+
+            [{'mux_path': mux_path,
+              'variant_id': variant_id,
+              'variant': dump_tree_nodes(original_variant)},
+             {'mux_path': [str, str, ...],
+              'variant_id': str,
+              'variant': [(str, [(str, str, object), ...])],
+             {'mux_path': ['/run/*'],
+              'variant_id': 'aaa-26c0'
+              'variant': [('/foo/aaa',
+                           [('/foo', 'bar', 'baz'),
+                            ('/foo/aaa', 'bbb', 'ccc')])]}
+             ...]
+
+        where `dump_tree_nodes` looks like::
+
+            [(node.path, environment_representation),
+             (node.path, [(path1, key1, value1), (path2, key2, value2), ...]),
+             ('/foo/aaa', [('/foo', 'bar', 'baz')])
+
+        :return: loadable Varianter representation
+        """
+        def dump_tree_node(node):
+            """
+            Turns TreeNode-like object into tuple(path, env_representation)
+            """
+            return (str(node.path),
+                    [(str(node.environment.origin[key].path), str(key), value)
+                     for key, value in node.environment.iteritems()])
+
+        if not self.is_parsed():
+            raise NotImplementedError("Dumping Varianter state before "
+                                      "multiplexation is not supported.")
+        variants = []
+        for variant in self.itertests():
+            safe_variant = {}
+            safe_variant["mux_path"] = [str(pth)
+                                        for pth in variant.get("mux_path")]
+            safe_variant["variant_id"] = str(variant.get("variant_id"))
+            safe_variant["variant"] = [dump_tree_node(_)
+                                       for _ in variant.get("variant", [])]
+            variants.append(safe_variant)
+
+        return variants
+
+    def load(self, state):
+        """
+        Load the variants state
+
+        Current implementation supports loading from a list of loadable
+        variants. It replaces the VariantDispatcher with fake implementation
+        which reports the loaded (and initialized) variants.
+
+        :param state: loadable Varianter representation
+        """
+        self.debug = False
+        self.node_class = tree.TreeNode
+        self._variant_plugins = FakeVariantDispatcher(state)
+        self._no_variants = sum(self._variant_plugins.map_method("__len__"))
+
     def itertests(self):
         """
         Yields all variants of all plugins

avocado/plugins/replay.py

@@ -254,14 +254,9 @@ class Replay(CLI):
                 LOG_UI.error('Source job variants data not found. Aborting.')
                 sys.exit(exit_codes.AVOCADO_FAIL)
             else:
-                # Ignore data manipulation. This is necessary, because
-                # we replaced the unparsed object with parsed one. There
-                # are other plugins running before/after this which might
-                # want to alter the variants object.
-                if args.avocado_variants.is_parsed():
-                    LOG_UI.warning("Using src job Mux data only, use "
-                                   "`--replay-ignore variants` to override "
-                                   "them.")
+                LOG_UI.warning("Using src job Mux data only, use "
+                               "`--replay-ignore variants` to override "
+                               "them.")
                 setattr(args, "avocado_variants", variants)
 
         # Extend "replay_test_status" of "INTERRUPTED" when --replay-resume

selftests/functional/test_replay_basic.py

@@ -60,8 +60,8 @@ class ReplayTests(unittest.TestCase):
         """
         Checks if all expected files are there.
         """
-        file_list = ['multiplex', 'config', 'test_references', 'pwd', 'args',
-                     'cmdline']
+        file_list = ['variants.json', 'config', 'test_references', 'pwd',
+                     'args.json', 'cmdline']
         for filename in file_list:
             path = os.path.join(self.jobdir, 'jobdata', filename)
             self.assertTrue(glob.glob(path))