From f9dbf7d7b34fcc7aafac5bd3c43500e5d5bb95f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Doktor?= Date: Tue, 6 Jun 2017 17:54:48 +0200 Subject: [PATCH] jobdata: Use json to store args MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently we dump args using pickle, which tries to pickle all objects. This could fail between versions or even between instances (eg. with different set of plugins). Let's just use safe json serialization replacing unsafe objects with None to avoid unnecessary crashes while still being able to make the safe information available. Signed-off-by: Lukáš Doktor --- avocado/core/jobdata.py | 11 +++++++++-- avocado/plugins/replay.py | 11 +++-------- selftests/functional/test_replay_basic.py | 4 ++-- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/avocado/core/jobdata.py b/avocado/core/jobdata.py index 40b7ff47..18820138 100644 --- a/avocado/core/jobdata.py +++ b/avocado/core/jobdata.py @@ -18,6 +18,7 @@ Record/retrieve job information import ast import glob +import json import os import pickle @@ -49,7 +50,7 @@ def record(args, logdir, mux, references=None, cmdline=None): TEST_REFERENCES_FILENAME_LEGACY) path_mux = os.path.join(base_dir, VARIANTS_FILENAME) path_pwd = os.path.join(base_dir, PWD_FILENAME) - path_args = os.path.join(base_dir, ARGS_FILENAME) + path_args = os.path.join(base_dir, ARGS_FILENAME + ".json") path_cmdline = os.path.join(base_dir, CMDLINE_FILENAME) if references: @@ -75,7 +76,7 @@ def record(args, logdir, mux, references=None, cmdline=None): os.fsync(pwd_file) with open(path_args, 'w') as args_file: - pickle.dump(args.__dict__, args_file, pickle.HIGHEST_PROTOCOL) + json.dump(args.__dict__, args_file, default=lambda x: None) args_file.flush() os.fsync(args_file) @@ -136,9 +137,15 @@ def retrieve_args(resultsdir): """ Retrieves the job args from the results directory. """ + recorded_args = _retrieve(resultsdir, ARGS_FILENAME + ".json") + if recorded_args: + with open(recorded_args, 'r') as args_file: + return json.load(args_file) recorded_args = _retrieve(resultsdir, ARGS_FILENAME) if recorded_args is None: return None + # old pickle-based dump + # TODO: Remove when 36lts is discontinued with open(recorded_args, 'r') as args_file: return pickle.load(args_file) diff --git a/avocado/plugins/replay.py b/avocado/plugins/replay.py index 0c4bff6f..75024da4 100644 --- a/avocado/plugins/replay.py +++ b/avocado/plugins/replay.py @@ -254,14 +254,9 @@ class Replay(CLI): LOG_UI.error('Source job variants data not found. Aborting.') sys.exit(exit_codes.AVOCADO_FAIL) else: - # Ignore data manipulation. This is necessary, because - # we replaced the unparsed object with parsed one. There - # are other plugins running before/after this which might - # want to alter the variants object. - if args.avocado_variants.is_parsed(): - LOG_UI.warning("Using src job Mux data only, use " - "`--replay-ignore variants` to override " - "them.") + LOG_UI.warning("Using src job Mux data only, use " + "`--replay-ignore variants` to override " + "them.") setattr(args, "avocado_variants", variants) # Extend "replay_test_status" of "INTERRUPTED" when --replay-resume diff --git a/selftests/functional/test_replay_basic.py b/selftests/functional/test_replay_basic.py index e3f4073a..e0129055 100644 --- a/selftests/functional/test_replay_basic.py +++ b/selftests/functional/test_replay_basic.py @@ -60,8 +60,8 @@ class ReplayTests(unittest.TestCase): """ Checks if all expected files are there. """ - file_list = ['variants', 'config', 'test_references', 'pwd', 'args', - 'cmdline'] + file_list = ['variants', 'config', 'test_references', 'pwd', + 'args.json', 'cmdline'] for filename in file_list: path = os.path.join(self.jobdir, 'jobdata', filename) self.assertTrue(glob.glob(path)) -- GitLab