From 905a45efe9d3990c89731d65b5876d0fd3386fcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Doktor?= Date: Tue, 4 Oct 2016 19:44:09 +0200 Subject: [PATCH] avocado.core.multiplexer: Define multiplexer API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch is a 1st step to independent multiplex plugin. It unifies the multiplexation, which is handled exclusively by the avocado.core.multiplexer.Mux object, which is initialized directly in argument parser as `args.mux`, instead of the `parser.default_avocado_params`. The idea is to provide unified database with simple interface: * data_inject - inject key/value[/path] data * data_merge - merge node/tree into the data Additionally it supports internal API to generate the params: * is_parsed - whether it was already initialized/filtered/... * get_number_of_tests - reports number of test*variants * itertests - iterate through tests All those changes are pickle-safe from the old object to the new object (not vice versa), therefor it's possible to use old Mux objects stored in `jobdata` to replay tests. This API is not 100% stable yet, but we plan to keep the one sided pickle compatibility into the future (no guarantee, but we'll do our best). Signed-off-by: Lukáš Doktor --- avocado/core/job.py | 13 ++--- avocado/core/jobdata.py | 2 +- avocado/core/multiplexer.py | 94 ++++++++++++++++++++++++++++++++---- avocado/core/parser.py | 7 ++- avocado/plugins/multiplex.py | 36 ++++---------- avocado/plugins/replay.py | 2 +- avocado/plugins/run.py | 14 ------ 7 files changed, 108 insertions(+), 60 deletions(-) diff --git a/avocado/core/job.py b/avocado/core/job.py index d3a3bc0a..64b458f2 100644 --- a/avocado/core/job.py +++ b/avocado/core/job.py @@ -446,14 +446,15 @@ class Job(object): "command.") raise exceptions.OptionValidationError(e_msg) - if isinstance(getattr(self.args, 'multiplex_files', None), - multiplexer.Mux): - mux = self.args.multiplex_files # pylint: disable=E1101 - else: + mux = getattr(self.args, "mux", None) + if mux is None: + mux = multiplexer.Mux() + if not mux.is_parsed(): # Mux not yet parsed, apply args try: - mux = multiplexer.Mux(self.args) + mux.parse(self.args) except (IOError, ValueError) as details: - raise exceptions.OptionValidationError(details) + raise exceptions.OptionValidationError("Unable to parse mux: " + "%s" % details) self.args.test_result_total = mux.get_number_of_tests(self.test_suite) self._make_test_result() diff --git a/avocado/core/jobdata.py b/avocado/core/jobdata.py index 14dc118e..fb182d19 100644 --- a/avocado/core/jobdata.py +++ b/avocado/core/jobdata.py @@ -100,7 +100,7 @@ def retrieve_urls(resultsdir): def retrieve_mux(resultsdir): """ - Retrieves the job multiplex from the results directory. + Retrieves the job Mux object from the results directory. """ recorded_mux = _retrieve(resultsdir, MUX_FILENAME) if recorded_mux is None: diff --git a/avocado/core/multiplexer.py b/avocado/core/multiplexer.py index ca753164..ab96661e 100644 --- a/avocado/core/multiplexer.py +++ b/avocado/core/multiplexer.py @@ -377,28 +377,104 @@ class AvocadoParam(object): yield (leaf.environment_origin[key].path, key, value) +def _report_mux_already_parsed(self, *args, **kwargs): + """ + Raises exception describing that `self.data` alteration is restricted + """ + raise RuntimeError("Mux already parsed, altering is restricted. %s %s" + % (args, kwargs)) + + class Mux(object): """ This is a multiplex object which multiplexes the test_suite. """ - def __init__(self, args): + def __init__(self): self._has_multiple_variants = None - mux_files = getattr(args, 'multiplex_files', None) + self.variants = None + self.data = tree.TreeNode() + self._mux_path = None + + def parse(self, args, debug=False): + """ + Apply options defined on the cmdline + + :param args: Parsed cmdline arguments + :param debug: Whether to debug the mux parsing + """ filter_only = getattr(args, 'filter_only', None) filter_out = getattr(args, 'filter_out', None) - if mux_files: - mux_tree = yaml2tree(mux_files) - else: # no variants - mux_tree = tree.TreeNode() - if getattr(args, 'default_avocado_params', None): - mux_tree.merge(args.default_avocado_params) - mux_tree = tree.apply_filters(mux_tree, filter_only, filter_out) + self._parse_basic_injects(args, debug) + mux_tree = tree.apply_filters(self.data, filter_only, filter_out) self.variants = MuxTree(mux_tree) self._mux_path = getattr(args, 'mux_path', None) if self._mux_path is None: self._mux_path = ['/run/*'] + # disable data alteration (and remove data as they are not useful) + self.data = None + self.data_inject = _report_mux_already_parsed + self.data_merge = _report_mux_already_parsed + + def _parse_basic_injects(self, args, debug): + """ + Inject data from the basic injects defined by Mux + + :param args: Parsed cmdline arguments + :param debug: Whether to debug the mux parsing + """ + # Merge the multiplex_files + multiplex_files = getattr(args, "multiplex_files", None) + if multiplex_files: + self.data_merge(yaml2tree(multiplex_files, debug=debug)) + + # FIXME: Backward compatibility params, to be removed when 36 LTS is + # discontinued + if (not getattr(args, "mux_skip_defaults", False) and + hasattr(args, "default_avocado_params")): + self.data_merge(args.default_avocado_params) + + # Extend default multiplex tree of --mux-inject values + for inject in getattr(args, "mux_inject", []): + entry = inject.split(':', 3) + if len(entry) < 2: + raise ValueError("key:entry pairs required, found only %s" + % (entry)) + elif len(entry) == 2: # key, entry + self.data_inject(*entry) + else: # path, key, entry + self.data_inject(key=entry[1], value=entry[2], path=entry[0]) + + def is_parsed(self): + """ + Reports whether the tree was already multiplexed + """ + return self.variants is not None + + def data_inject(self, key, value, path=None): + """ + Inject entry to the mux tree (params database) + + :param key: Key to which we'd like to assign the value + :param value: The key's value + :param path: Optional path to the node to which we assign the value, + by default '/'. + """ + if path: + node = self.data.get_node(path, True) + else: + node = self.data + node.value[key] = value + + def data_merge(self, tree): + """ + Merge tree into the mux tree (params database) + + :param tree: Tree to be merged into this database. + :type tree: :class:`avocado.core.tree.TreeNode` + """ + self.data.merge(tree) def get_number_of_tests(self, test_suite): """ diff --git a/avocado/core/parser.py b/avocado/core/parser.py index d167d450..cb38fc25 100644 --- a/avocado/core/parser.py +++ b/avocado/core/parser.py @@ -12,7 +12,6 @@ # Copyright: Red Hat Inc. 2013-2014 # Author: Ruda Moura - """ Avocado application command line parsing. """ @@ -21,8 +20,9 @@ import argparse import logging from . import exit_codes -from . import tree +from . import multiplexer from . import settings +from . import tree from .output import BUILTIN_STREAMS, BUILTIN_STREAM_SETS from .version import VERSION @@ -125,6 +125,9 @@ class Parser(object): dest='subcommand') # Allow overriding default params by plugins + self.args.mux = multiplexer.Mux() + # FIXME: Backward compatibility params, to be removed when 36 LTS is + # discontinued self.args.default_avocado_params = tree.TreeNode() def finish(self): diff --git a/avocado/plugins/multiplex.py b/avocado/plugins/multiplex.py index bfd95afa..ae2ee253 100644 --- a/avocado/plugins/multiplex.py +++ b/avocado/plugins/multiplex.py @@ -33,7 +33,6 @@ class Multiplex(CLICmd): def __init__(self, *args, **kwargs): super(Multiplex, self).__init__(*args, **kwargs) - self._from_args_tree = tree.TreeNode() def configure(self, parser): if multiplexer.MULTIPLEX_CAPABLE is False: @@ -48,7 +47,8 @@ class Multiplex(CLICmd): parser.add_argument('--filter-out', nargs='*', default=[], help='Filter out path(s) from multiplexing') - parser.add_argument('--system-wide', action='store_true', + parser.add_argument('--system-wide', action='store_false', + default=True, dest="mux-skip-defaults", help="Combine the files with the default " "tree.") parser.add_argument('-c', '--contents', action='store_true', @@ -68,21 +68,7 @@ class Multiplex(CLICmd): tree_parser.add_argument('-i', '--inherit', action="store_true", help="Show the inherited values") - def _activate(self, args): - # Extend default multiplex tree of --env values - for value in getattr(args, "mux_inject", []): - value = value.split(':', 2) - if len(value) < 2: - raise ValueError("key:value pairs required, found only %s" - % (value)) - elif len(value) == 2: - self._from_args_tree.value[value[0]] = value[1] - else: - node = self._from_args_tree.get_node(value[0], True) - node.value[value[1]] = value[2] - def run(self, args): - self._activate(args) log = logging.getLogger("avocado.app") err = None if args.tree and args.debug: @@ -92,16 +78,13 @@ class Multiplex(CLICmd): if err: log.error(err) sys.exit(exit_codes.AVOCADO_FAIL) + mux = multiplexer.Mux() + root = mux.data try: - mux_tree = multiplexer.yaml2tree(args.multiplex_files, - args.filter_only, args.filter_out, - args.debug) - except IOError as details: - log.error(details.strerror) + mux.parse(args, debug=args.debug) + except (IOError, ValueError) as details: + log.error("Unable to parse mux: %s", details) sys.exit(exit_codes.AVOCADO_JOB_FAIL) - if args.system_wide: - mux_tree.merge(args.default_avocado_params) - mux_tree.merge(self._from_args_tree) if args.tree: if args.contents: verbose = 1 @@ -111,12 +94,11 @@ class Multiplex(CLICmd): verbose += 2 use_utf8 = settings.get_value("runner.output", "utf8", key_type=bool, default=None) - log.debug(tree.tree_view(mux_tree, verbose, use_utf8)) + log.debug(tree.tree_view(root, verbose, use_utf8)) sys.exit(exit_codes.AVOCADO_ALL_OK) - variants = multiplexer.MuxTree(mux_tree) log.info('Variants generated:') - for (index, tpl) in enumerate(variants): + for (index, tpl) in enumerate(mux.variants): if not args.debug: paths = ', '.join([x.path for x in tpl]) else: diff --git a/avocado/plugins/replay.py b/avocado/plugins/replay.py index 7bfcfa38..76a5b4d3 100644 --- a/avocado/plugins/replay.py +++ b/avocado/plugins/replay.py @@ -217,7 +217,7 @@ class Replay(CLI): log.error('Source job multiplex data not found. Aborting.') sys.exit(exit_codes.AVOCADO_JOB_FAIL) else: - setattr(args, "multiplex_files", mux) + setattr(args, "mux", mux) if args.replay_teststatus: replay_map = self._create_replay_map(resultsdir, diff --git a/avocado/plugins/run.py b/avocado/plugins/run.py index 07d2ead3..72357ada 100644 --- a/avocado/plugins/run.py +++ b/avocado/plugins/run.py @@ -150,19 +150,6 @@ class Run(CLICmd): help="Inject [path:]key:node values into the " "final multiplex tree.") - def _activate(self, args): - # Extend default multiplex tree of --mux_inject values - for value in getattr(args, "mux_inject", []): - value = value.split(':', 2) - if len(value) < 2: - raise ValueError("key:value pairs required, found only %s" - % (value)) - elif len(value) == 2: - args.default_avocado_params.value[value[0]] = value[1] - else: - node = args.default_avocado_params.get_node(value[0], True) - node.value[value[1]] = value[2] - def run(self, args): """ Run test modules or simple tests. @@ -170,7 +157,6 @@ class Run(CLICmd): :param args: Command line args received from the run subparser. """ log = logging.getLogger("avocado.app") - self._activate(args) if args.unique_job_id is not None: try: int(args.unique_job_id, 16) -- GitLab