From d5076fac4891ed886095a27bc884815700224e3c Mon Sep 17 00:00:00 2001 From: Amador Pahim Date: Wed, 14 Mar 2018 12:44:26 +0100 Subject: [PATCH] test results: optimize params information Currently we propagate to test results the AvacadoParams object used by the test. Holding that object in the test results makes the Avocado process to consume a lot of memory. For instance, in an Avocado job with 5000 tests (passtest.py), with Avocado-VT in place, the Job object grows up from 59MB before running the tests to 274MB after running the tests, a 215MB growth:: (Pdb) pre_tests Partition of a set of 440754 objects. Total size = 59066528 bytes. (Pdb) post_tests Partition of a set of 1570821 objects. Total size = 274340184 bytes. (Pdb) growth Partition of a set of 1130073 objects. Total size = 215274464 bytes. Since the 'params' item is important for both the HTML report and the ResultsDB report, instead of dropping the 'params' item from the results, this patch parses the AvocadoParams, populating the test results with a list of tuples, each one containing the 'path', 'key' and 'value' per parameter basis. This improves the situation a lot. The same Avocado Job will now only grow up from 59MB to 89MB:: (Pdb) pre_tests Partition of a set of 440747 objects. Total size = 59066424 bytes. (Pdb) post_tests Partition of a set of 840812 objects. Total size = 89619904 bytes. (Pdb) growth Partition of a set of 400071 objects. Total size = 30554288 bytes. The HTML and ResultsDB plugins were adapted accordingly. Signed-off-by: Amador Pahim --- avocado/core/test.py | 5 ++++- optional_plugins/html/avocado_result_html/__init__.py | 2 +- optional_plugins/resultsdb/avocado_resultsdb/__init__.py | 5 ++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/avocado/core/test.py b/avocado/core/test.py index 74ce57ed..9a16d0b3 100644 --- a/avocado/core/test.py +++ b/avocado/core/test.py @@ -60,7 +60,7 @@ TEST_STATE_ATTRIBUTES = ('name', 'logdir', 'logfile', 'status', 'running', 'paused', 'time_start', 'time_elapsed', 'time_end', 'fail_reason', 'fail_class', 'traceback', - 'params', 'timeout', 'whiteboard') + 'timeout', 'whiteboard') class RawFileHandler(logging.FileHandler): @@ -630,6 +630,9 @@ class Test(unittest.TestCase, TestData): state['class_name'] = self.__class__.__name__ state['job_logdir'] = self.job.logdir state['job_unique_id'] = self.job.unique_id + state['params'] = [(path, key, value) + for path, key, value + in self.params.iteritems()] return state def _register_log_file_handler(self, logger, formatter, filename, diff --git a/optional_plugins/html/avocado_result_html/__init__.py b/optional_plugins/html/avocado_result_html/__init__.py index 9da633f6..f7a9e870 100644 --- a/optional_plugins/html/avocado_result_html/__init__.py +++ b/optional_plugins/html/avocado_result_html/__init__.py @@ -129,7 +129,7 @@ class ReportModel(object): params = '' try: parameters = 'Params:\n' - for path, key, value in tst['params'].iteritems(): + for path, key, value in tst['params']: parameters += ' %s:%s => %s\n' % (path, key, value) except KeyError: pass diff --git a/optional_plugins/resultsdb/avocado_resultsdb/__init__.py b/optional_plugins/resultsdb/avocado_resultsdb/__init__.py index d048bd56..34bc2caa 100644 --- a/optional_plugins/resultsdb/avocado_resultsdb/__init__.py +++ b/optional_plugins/resultsdb/avocado_resultsdb/__init__.py @@ -114,9 +114,8 @@ class ResultsdbResultEvent(ResultEvents): 'status': state['status']} params = {} - for param in iteritems(state['params']): - params['param %s' % param[1]] = '%s (path: %s)' % (param[2], - param[0]) + for path, key, value in state['params']: + params['param %s' % key] = '%s (path: %s)' % (value, path) data.update(params) self.rdbapi.create_result(outcome, name, group, note, ref_url, **data) -- GitLab