From 60e0b72f6c5d0d6d9e699cb1bd22f54d3f0c771c Mon Sep 17 00:00:00 2001 From: Cleber Rosa Date: Thu, 21 Jun 2018 00:34:51 -0400 Subject: [PATCH] CIT Varianter Implementation of the CIT varianter optional plugin. Signed-off-by: Amador Pahim Signed-off-by: Cleber Rosa --- docs/source/optional_plugins/index.rst | 1 + .../source/optional_plugins/varianter_cit.rst | 183 ++++++++++ examples/varianter_cit/params.ini | 6 + optional_plugins/varianter_cit/MANIFEST.in | 1 + optional_plugins/varianter_cit/VERSION | 1 + optional_plugins/varianter_cit/__init__.py | 0 .../avocado_varianter_cit/__init__.py | 322 ++++++++++++++++++ optional_plugins/varianter_cit/setup.py | 37 ++ .../varianter_cit/tests/__init__.py | 0 .../varianter_cit/tests/test_functional.py | 29 ++ .../varianter_cit/tests/test_unit.py | 41 +++ selftests/run | 4 +- 12 files changed, 624 insertions(+), 1 deletion(-) create mode 100644 docs/source/optional_plugins/varianter_cit.rst create mode 100644 examples/varianter_cit/params.ini create mode 100644 optional_plugins/varianter_cit/MANIFEST.in create mode 100644 optional_plugins/varianter_cit/VERSION create mode 100644 optional_plugins/varianter_cit/__init__.py create mode 100644 optional_plugins/varianter_cit/avocado_varianter_cit/__init__.py create mode 100644 optional_plugins/varianter_cit/setup.py create mode 100644 optional_plugins/varianter_cit/tests/__init__.py create mode 100644 optional_plugins/varianter_cit/tests/test_functional.py create mode 100644 optional_plugins/varianter_cit/tests/test_unit.py diff --git a/docs/source/optional_plugins/index.rst b/docs/source/optional_plugins/index.rst index f5574b39..82a8855d 100644 --- a/docs/source/optional_plugins/index.rst +++ b/docs/source/optional_plugins/index.rst @@ -15,6 +15,7 @@ optional plugins: robot varianter_yaml_to_mux varianter_pict + varianter_cit yaml_loader golang glib diff --git a/docs/source/optional_plugins/varianter_cit.rst b/docs/source/optional_plugins/varianter_cit.rst new file mode 100644 index 00000000..6ba0316a --- /dev/null +++ b/docs/source/optional_plugins/varianter_cit.rst @@ -0,0 +1,183 @@ +==================== +CIT Varianter Plugin +==================== + +This plugin is an implementation of a Combinatorial Interaction +Testing algorithm for the Avocado varianter functionality. It +generates an optimal number of variants, which in turn become +different test scenarios. + +.. note:: The publication by Ahmed, Bestoun S., Kamal Z. Zamli, and + Chee Peng Lim, entitled `“Application of particle swarm + optimization to uniform and variable strength covering array + construction” `__, + Applied Soft Computing, 12(4), 2012, pp. 1330-1347, contains + the basis for the algorithm and implementation of this + feature. + +Files +===== + +- ``optional_plugins/varianter_cit/avocado_varianter_cit/__init__.py``: The + plugin implementation. + +- ``examples/varianter_cit/params.ini``: An example parameters file. + +Usage +===== + +To see the variants generated by this demo implementation, execute:: + + $ avocado variants --cit-parameter-file examples/varianter_cit/params.ini + CIT Variants (14): + Variant gold-triangle-solid-aluminum-cathodic: /run + Variant red-circle-gas-leather-cathodic: /run + Variant gold-square-gas-plastic-anodic: /run + Variant black-circle-liquid-aluminum-anodic: /run + Variant green-triangle-liquid-leather-anodic: /run + Variant green-circle-solid-plastic-cathodic: /run + Variant black-square-solid-leather-cathodic: /run + Variant red-square-liquid-aluminum-anodic: /run + Variant red-triangle-liquid-plastic-cathodic: /run + Variant black-triangle-gas-plastic-anodic: /run + Variant green-square-gas-aluminum-anodic: /run + Variant gold-circle-solid-leather-anodic: /run + Variant gold-triangle-liquid-plastic-anodic: /run + Variant red-circle-solid-aluminum-cathodic: /run + +.. note:: The exact variants generated are not guaranteed to be the same + across executions. + +You can enable more verbosity, making each variant to show its content:: + + $ avocado variants --cit-parameter-file examples/varianter_cit/params.ini -c + CIT Variants (15): + + Variant black-square-solid-leather-cathodic: /run + /:coating => cathodic + /:color => black + /:material => leather + /:shape => square + /:state => solid + + Variant red-circle-gas-aluminum-cathodic: /run + /:coating => cathodic + /:color => red + /:material => aluminum + /:shape => circle + /:state => gas + + Variant red-triangle-liquid-plastic-anodic: /run + /:coating => anodic + /:color => red + /:material => plastic + /:shape => triangle + /:state => liquid + + Variant green-square-gas-aluminum-anodic: /run + /:coating => anodic + /:color => green + /:material => aluminum + /:shape => square + /:state => gas + + Variant gold-circle-solid-leather-anodic: /run + /:coating => anodic + /:color => gold + /:material => leather + /:shape => circle + /:state => solid + + Variant green-triangle-solid-plastic-cathodic: /run + /:coating => cathodic + /:color => green + /:material => plastic + /:shape => triangle + /:state => solid + + Variant gold-square-liquid-plastic-cathodic: /run + /:coating => cathodic + /:color => gold + /:material => plastic + /:shape => square + /:state => liquid + + Variant black-triangle-gas-leather-anodic: /run + /:coating => anodic + /:color => black + /:material => leather + /:shape => triangle + /:state => gas + + Variant black-circle-liquid-plastic-anodic: /run + /:coating => anodic + /:color => black + /:material => plastic + /:shape => circle + /:state => liquid + + Variant gold-triangle-gas-aluminum-anodic: /run + /:coating => anodic + /:color => gold + /:material => aluminum + /:shape => triangle + /:state => gas + + Variant green-circle-liquid-leather-anodic: /run + /:coating => anodic + /:color => green + /:material => leather + /:shape => circle + /:state => liquid + + Variant red-square-solid-aluminum-anodic: /run + /:coating => anodic + /:color => red + /:material => aluminum + /:shape => square + /:state => solid + + Variant black-triangle-liquid-aluminum-anodic: /run + /:coating => anodic + /:color => black + /:material => aluminum + /:shape => triangle + /:state => liquid + + Variant red-square-gas-plastic-anodic: /run + /:coating => anodic + /:color => red + /:material => plastic + /:shape => square + /:state => gas + + Variant red-square-liquid-leather-cathodic: /run + /:coating => cathodic + /:color => red + /:material => leather + /:shape => square + /:state => liquid + +To execute tests with those combinations use:: + + $ avocado run passtest.py --cit-parameter-file examples/varianter_cit/params.ini + JOB ID : 6abd9e9f1ff9ed33a353ca8f3ef845cd4cc404a5 + JOB LOG : $HOME/avocado/job-results/job-2018-07-23T08.46-6abd9e9/job.log + (01/15) passtest.py:PassTest.test;gold-circle-gas-plastic-cathodic: PASS (0.06 s) + (02/15) passtest.py:PassTest.test;green-square-solid-plastic-anodic: PASS (0.02 s) + (03/15) passtest.py:PassTest.test;black-triangle-liquid-aluminum-anodic: PASS (0.02 s) + (04/15) passtest.py:PassTest.test;red-triangle-solid-leather-cathodic: PASS (0.02 s) + (05/15) passtest.py:PassTest.test;red-square-liquid-aluminum-cathodic: PASS (0.02 s) + (06/15) passtest.py:PassTest.test;green-circle-gas-leather-anodic: PASS (0.02 s) + (07/15) passtest.py:PassTest.test;gold-circle-solid-aluminum-anodic: PASS (0.02 s) + (08/15) passtest.py:PassTest.test;black-square-gas-leather-cathodic: PASS (0.02 s) + (09/15) passtest.py:PassTest.test;red-circle-liquid-plastic-anodic: PASS (0.02 s) + (10/15) passtest.py:PassTest.test;green-triangle-gas-aluminum-cathodic: PASS (0.02 s) + (11/15) passtest.py:PassTest.test;gold-triangle-liquid-leather-cathodic: PASS (0.02 s) + (12/15) passtest.py:PassTest.test;black-circle-solid-plastic-anodic: PASS (0.02 s) + (13/15) passtest.py:PassTest.test;red-triangle-gas-plastic-cathodic: PASS (0.02 s) + (14/15) passtest.py:PassTest.test;gold-square-liquid-leather-anodic: PASS (0.02 s) + (15/15) passtest.py:PassTest.test;green-circle-liquid-aluminum-anodic: PASS (0.02 s) + RESULTS : PASS 15 | ERROR 0 | FAIL 0 | SKIP 0 | WARN 0 | INTERRUPT 0 | CANCEL 0 + JOB TIME : 1.21 s + JOB HTML : $HOME/avocado/job-results/job-2018-07-23T08.46-6abd9e9/results.html diff --git a/examples/varianter_cit/params.ini b/examples/varianter_cit/params.ini new file mode 100644 index 00000000..32e9c590 --- /dev/null +++ b/examples/varianter_cit/params.ini @@ -0,0 +1,6 @@ +[parameters] +color: black, gold, red, green +shape: square, triangle, circle +state: liquid, solid, gas +material: leather, plastic, aluminum +coating: anodic, cathodic diff --git a/optional_plugins/varianter_cit/MANIFEST.in b/optional_plugins/varianter_cit/MANIFEST.in new file mode 100644 index 00000000..ceeea233 --- /dev/null +++ b/optional_plugins/varianter_cit/MANIFEST.in @@ -0,0 +1 @@ +include VERSION diff --git a/optional_plugins/varianter_cit/VERSION b/optional_plugins/varianter_cit/VERSION new file mode 100644 index 00000000..e630b42c --- /dev/null +++ b/optional_plugins/varianter_cit/VERSION @@ -0,0 +1 @@ +63.0 diff --git a/optional_plugins/varianter_cit/__init__.py b/optional_plugins/varianter_cit/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/optional_plugins/varianter_cit/avocado_varianter_cit/__init__.py b/optional_plugins/varianter_cit/avocado_varianter_cit/__init__.py new file mode 100644 index 00000000..e3213054 --- /dev/null +++ b/optional_plugins/varianter_cit/avocado_varianter_cit/__init__.py @@ -0,0 +1,322 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See LICENSE for more details. +# Authors: Amador Pahim +# Bestoun S. Ahmed +# Cleber Rosa + +import copy +import itertools +import os +import random +import sys + +from six import iteritems +from six.moves import configparser +from six.moves import zip + +from avocado.core import exit_codes +from avocado.core.output import LOG_UI +from avocado.core.plugin_interfaces import CLI +from avocado.core.plugin_interfaces import Varianter +from avocado.core.tree import TreeNode + + +class VarianterCitCLI(CLI): + + """ + CIT Varianter options + """ + + name = 'cit' + description = "CIT Varianter options for the 'run' subcommand" + + def configure(self, parser): + + for name in ("run", "variants"): + subparser = parser.subcommands.choices.get(name, None) + if subparser is None: + continue + cit = subparser.add_argument_group('CIT varianter options') + cit.add_argument('--cit-parameter-file', metavar='PATH', + help="Paths to a parameter file") + cit.add_argument('--cit-parameter-path', metavar='PATH', + default='/run', + help=('Default path for parameters generated ' + 'on the CIT variants')) + cit.add_argument('--cit-order-of-combinations', + metavar='ORDER', type=int, default=2, + help=("Order of combinations. Defaults to " + "%(default)s, maximum number is specific " + "to parameter file content")) + + def run(self, args): + pass + + +class VarianterCit(Varianter): + + """ + Processes the parameters file into variants + """ + + name = 'cit' + description = "CIT Varianter" + + def initialize(self, args): + self.variants = None + + cit_parameter_file = getattr(args, "cit_parameter_file", None) + if cit_parameter_file is None: + return + else: + cit_parameter_file = os.path.expanduser(cit_parameter_file) + if not os.access(cit_parameter_file, os.R_OK): + LOG_UI.error("parameter file '%s' could not be found or " + "is not readable", cit_parameter_file) + self.error_exit(args) + + self.parameter_path = getattr(args, "cit_parameter_path") + config = configparser.ConfigParser() + try: + config.read(cit_parameter_file) + except Exception as details: + LOG_UI.error("Cannot parse parameter file: %s", details) + self.error_exit(args) + + parameters = [(key, value.split(', ')) + for key, value in config.items('parameters')] + order = args.cit_order_of_combinations + cit = Cit(parameters, order) + self.headers, self.variants = cit.combine() + + @staticmethod + def error_exit(args): + if args.subcommand == 'run': + sys.exit(exit_codes.AVOCADO_JOB_FAIL) + else: + sys.exit(exit_codes.AVOCADO_FAIL) + + def __iter__(self): + if self.variants is None: + return + + variant_ids = [] + for variant in self.variants: + variant_ids.append("-".join([variant.get(key) + for key in self.headers])) + + for vid, variant in zip(variant_ids, self.variants): + yield {"variant_id": vid, + "variant": TreeNode('', variant), + "paths": self.parameter_path} + + def __len__(self): + return sum(1 for _ in self.variants) if self.variants else 0 + + def update_defaults(self, defaults): + pass + + def to_str(self, summary, variants, **kwargs): + """ + Return human readable representation + + The summary/variants accepts verbosity where 0 means silent and + maximum is up to the plugin. + + :param summary: How verbose summary to output (int) + :param variants: How verbose list of variants to output (int) + :param kwargs: Other free-form arguments + :rtype: str + """ + if not self.variants: + return "" + + out = [] + verbose = variants > 1 + out.append("CIT Variants (%i):" % len(self)) + for variant in self: + out.append('%sVariant %s: %s' % ('\n' if verbose else '', + variant["variant_id"], + self.parameter_path)) + if not verbose: + continue + env = set() + for node in variant["variant"]: + for key, value in iteritems(node.environment): + origin = node.environment.origin[key].path + env.add(("%s:%s" % (origin, key), str(value))) + if not env: + return out + fmt = ' %%-%ds => %%s' % max([len(_[0]) for _ in env]) + for record in sorted(env): + out.append(fmt % record) + + return "\n".join(out) + + +class Cit(object): + + MATRIX_ROW_SIZE = 20 + MAX_ITERATIONS = 15 + + def __init__(self, parameters, order): + # Parameters come as ('key', ['value1', 'value2', 'value3']) + self.parameters = parameters + # Length (number of values) for each parameter + self.parameters_length = [len(param[1]) for param in self.parameters] + # Order of combinations + self.order = min(order, len(parameters)) + self.hash_table = {} + + def combine(self): + """ + Computes the combination of parameters + + :returns: headers (list of parameters keys) and combinations (list of + dictionaries. Each dictionary represents a combination of + parameters. + :rtype: tuple + """ + self.create_interaction_hash_table() + final_list = self.create_final_list() + + headers = [item[0] for item in self.parameters] + result = [[self.parameters[i][1][combination[i]] + for i in range(len(combination))] + for combination in final_list] + combinations = [] + for combination in result: + combinations.append(dict(zip(headers, combination))) + + return headers, combinations + + def create_interaction_hash_table(self): + for c in itertools.combinations(range(len(self.parameters_length)), self.order): + self.hash_table[c] = self.get_iteration(c) + + def create_final_list(self): + final_list = [] + while len(self.hash_table) != 0: + iterations = 0 + previous_test_case = [] + previous_remove_list = {} + previous_weight = 0 + + while iterations < self.MAX_ITERATIONS: + max_width = len(self.hash_table) + matrix = self.create_random_matrix() + remove_list = {} + for i in matrix: + width = self.get_weight(i, remove_list) + if width == 0 or width <= previous_weight: + remove_list.clear() + continue + elif width == max_width: + final_list.append(i) + self.remove_from_hash_table(remove_list) + previous_test_case = [] + previous_remove_list = {} + continue + elif width > previous_weight: + previous_weight = width + previous_test_case = i + previous_remove_list = dict(remove_list) + remove_list.clear() + iterations += 1 + if len(previous_test_case) != 0: + previous_remove_list = self.neighborhood_search( + previous_test_case, previous_weight, + max_width, previous_remove_list) + final_list.append(previous_test_case) + self.remove_from_hash_table(previous_remove_list) + return final_list + + def get_iteration(self, parameter_combination): + parameters_array = [] + for c in parameter_combination: + array = range(self.parameters_length[c]) + parameters_array.append(array) + iterations = {} + for i in itertools.product(*parameters_array): + iterations[i] = tuple(i) + return iterations + + def get_weight(self, test_case, remove_list): + weight = 0 + for i in self.hash_table: + iteration = tuple(test_case[j] for j in i) + try: + value = self.hash_table[i][iteration] + weight += 1 + remove_list[i] = value + except KeyError: + continue + + return weight + + def remove_from_hash_table(self, remove_list): + for i in remove_list: + del self.hash_table[i][remove_list[i]] + if len(self.hash_table[i]) == 0: + self.hash_table.pop(i) + remove_list.clear() + + def create_random_matrix(self): + matrix = [] + for _ in range(self.MATRIX_ROW_SIZE): + row = [] + for j in self.parameters_length: + row.append(random.randint(0, j-1)) + matrix.append(row) + return matrix + + def neighborhood_search(self, test_case, width, max_width, remove_list): + neighborhood = list(test_case) + neighborhood_remove_list = {} + for i in range(len(test_case)): + # neighborhood +1 + if (neighborhood[i] + 1) == self.parameters_length[i]: + neighborhood[i] = 0 + else: + neighborhood[i] += 1 + neighborhood_width = self.get_weight(neighborhood, neighborhood_remove_list) + if neighborhood_width > width: + width = neighborhood_width + remove_list = copy.deepcopy(neighborhood_remove_list) + del test_case[:] + for j in neighborhood: + test_case.append(j) + if neighborhood_width == max_width: + return remove_list + if neighborhood[i] == 0: + neighborhood[i] = self.parameters_length[i] - 1 + else: + neighborhood[i] -= 1 + + # neighborhood -1 + if neighborhood[i] == 0: + neighborhood[i] = self.parameters_length[i] - 1 + else: + neighborhood[i] -= 1 + neighborhood_width = self.get_weight(neighborhood, neighborhood_remove_list) + if neighborhood_width > width: + width = neighborhood_width + remove_list = copy.deepcopy(neighborhood_remove_list) + del test_case[:] + for j in neighborhood: + test_case.append(j) + if neighborhood_width == max_width: + return remove_list + if (neighborhood[i] + 1) == self.parameters_length[i]: + neighborhood[i] = 0 + else: + neighborhood[i] += 1 + return remove_list diff --git a/optional_plugins/varianter_cit/setup.py b/optional_plugins/varianter_cit/setup.py new file mode 100644 index 00000000..3e02653e --- /dev/null +++ b/optional_plugins/varianter_cit/setup.py @@ -0,0 +1,37 @@ +#!/bin/env python +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See LICENSE for more details. +# Authors: Amador Pahim +# Bestoun S. Ahmed +# Cleber Rosa + + +from setuptools import setup, find_packages + + +setup(name='avocado-framework-plugin-varianter-cit', + description='Varianter with combinatorial capabilities', + version=open("VERSION", "r").read().strip(), + author='Avocado Developers', + author_email='avocado-devel@redhat.com', + url='http://avocado-framework.github.io/', + packages=find_packages(), + include_package_data=True, + install_requires=['avocado-framework', ], + test_suite='tests', + entry_points={ + 'avocado.plugins.cli': [ + 'varianter_cit = avocado_varianter_cit:VarianterCitCLI', + ], + "avocado.plugins.varianter": [ + "varianter_cit = avocado_varianter_cit:VarianterCit", + ]} + ) diff --git a/optional_plugins/varianter_cit/tests/__init__.py b/optional_plugins/varianter_cit/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/optional_plugins/varianter_cit/tests/test_functional.py b/optional_plugins/varianter_cit/tests/test_functional.py new file mode 100644 index 00000000..d84e9a07 --- /dev/null +++ b/optional_plugins/varianter_cit/tests/test_functional.py @@ -0,0 +1,29 @@ +import os +import unittest + +from avocado.utils import process + + +basedir = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', '..') +basedir = os.path.abspath(basedir) + +AVOCADO = os.environ.get("UNITTEST_AVOCADO_CMD", "./scripts/avocado") + + +class Variants(unittest.TestCase): + + def test_max_variants(self): + os.chdir(basedir) + cmd_line = ( + '{0} variants --cit-order-of-combinations=5 ' + '--cit-parameter-file examples/varianter_cit/params.ini' + ).format(AVOCADO) + os.chdir(basedir) + result = process.run(cmd_line) + lines = result.stdout.splitlines() + self.assertEqual(b'CIT Variants (216):', lines[0]) + self.assertEqual(217, len(lines)) + + +if __name__ == '__main__': + unittest.main() diff --git a/optional_plugins/varianter_cit/tests/test_unit.py b/optional_plugins/varianter_cit/tests/test_unit.py new file mode 100644 index 00000000..807a5c0e --- /dev/null +++ b/optional_plugins/varianter_cit/tests/test_unit.py @@ -0,0 +1,41 @@ +import unittest + +from avocado_varianter_cit import Cit + + +class CitTest(unittest.TestCase): + + PARAMS = [('key1', ['x1', 'x2', 'x3']), ('key2', ['y1', 'y2', 'y3']), + ('key3', ['z1', 'z2', 'z3']), ('key4', ['w1', 'w2', 'w3'])] + + def test_orders(self): + for i in range(len(self.PARAMS)): + cit = Cit(self.PARAMS, i+1) + headers, _ = cit.combine() + # headers should always be the same, no matter the order + # of combinations + self.assertEqual(headers, ['key1', 'key2', 'key3', 'key4']) + + def test_number_of_combinations(self): + # the algorithm doesn't allow us to know beforehand, for a + # reason, the precise number of combinations that will be + # computed. still, we can check for the mininum number of + # combinations that should be produced + cit = Cit(self.PARAMS, 1) + self.assertEqual(len(cit.combine()[1]), 3) + cit = Cit(self.PARAMS, 2) + self.assertGreaterEqual(len(cit.combine()[1]), 9) + cit = Cit(self.PARAMS, 3) + self.assertGreaterEqual(len(cit.combine()[1]), 27) + + def test_max_order(self): + # test that with a order equal or larger than the number of + # keys, we have a predictable number of combinations + cit = Cit(self.PARAMS, 4) + self.assertEqual(len(cit.combine()[1]), 81) + cit = Cit(self.PARAMS, 10) + self.assertEqual(len(cit.combine()[1]), 81) + + +if __name__ == '__main__': + unittest.main() diff --git a/selftests/run b/selftests/run index bd263944..f365223a 100755 --- a/selftests/run +++ b/selftests/run @@ -38,7 +38,9 @@ def test_suite(): ('avocado-framework-plugin-runner-remote', 'runner_remote'), ('avocado-framework-plugin-runner-vm', - 'runner_vm')) + 'runner_vm'), + ('avocado-framework-plugin-varianter-cit', + 'varianter_cit')) for plugin_name, plugin_dir in plugins: if plugin_available(plugin_name): path = os.path.join(basedir, 'optional_plugins', -- GitLab