CIT Varianter

Implementation of the CIT varianter optional plugin.
Signed-off-by: NAmador Pahim <amador@apahim.org>
Signed-off-by: NCleber Rosa <crosa@redhat.com>

docs/source/optional_plugins/index.rst

@@ -15,6 +15,7 @@ optional plugins:
    robot
    varianter_yaml_to_mux
    varianter_pict
+   varianter_cit
    yaml_loader
    golang
    glib

docs/source/optional_plugins/varianter_cit.rst

+====================
+CIT Varianter Plugin
+====================
+
+This plugin is an implementation of a Combinatorial Interaction
+Testing algorithm for the Avocado varianter functionality.  It
+generates an optimal number of variants, which in turn become
+different test scenarios.
+
+.. note:: The publication by Ahmed, Bestoun S., Kamal Z. Zamli, and
+          Chee Peng Lim, entitled `“Application of particle swarm
+          optimization to uniform and variable strength covering array
+          construction” <http://booksc.org/book/13151468/aa9fc9>`__,
+          Applied Soft Computing, 12(4), 2012, pp. 1330-1347, contains
+          the basis for the algorithm and implementation of this
+          feature.
+
+Files
+=====
+
+- ``optional_plugins/varianter_cit/avocado_varianter_cit/__init__.py``: The
+  plugin implementation.
+
+- ``examples/varianter_cit/params.ini``: An example parameters file.
+
+Usage
+=====
+
+To see the variants generated by this demo implementation, execute::
+
+    $ avocado variants --cit-parameter-file examples/varianter_cit/params.ini
+    CIT Variants (14):
+    Variant gold-triangle-solid-aluminum-cathodic:    /run
+    Variant red-circle-gas-leather-cathodic:    /run
+    Variant gold-square-gas-plastic-anodic:    /run
+    Variant black-circle-liquid-aluminum-anodic:    /run
+    Variant green-triangle-liquid-leather-anodic:    /run
+    Variant green-circle-solid-plastic-cathodic:    /run
+    Variant black-square-solid-leather-cathodic:    /run
+    Variant red-square-liquid-aluminum-anodic:    /run
+    Variant red-triangle-liquid-plastic-cathodic:    /run
+    Variant black-triangle-gas-plastic-anodic:    /run
+    Variant green-square-gas-aluminum-anodic:    /run
+    Variant gold-circle-solid-leather-anodic:    /run
+    Variant gold-triangle-liquid-plastic-anodic:    /run
+    Variant red-circle-solid-aluminum-cathodic:    /run
+
+.. note:: The exact variants generated are not guaranteed to be the same
+          across executions.
+
+You can enable more verbosity, making each variant to show its content::
+
+    $ avocado variants --cit-parameter-file examples/varianter_cit/params.ini -c
+    CIT Variants (15):
+
+    Variant black-square-solid-leather-cathodic:    /run
+        /:coating     => cathodic
+        /:color       => black
+        /:material    => leather
+        /:shape       => square
+        /:state       => solid
+
+    Variant red-circle-gas-aluminum-cathodic:    /run
+        /:coating     => cathodic
+        /:color       => red
+        /:material    => aluminum
+        /:shape       => circle
+        /:state       => gas
+
+    Variant red-triangle-liquid-plastic-anodic:    /run
+        /:coating     => anodic
+        /:color       => red
+        /:material    => plastic
+        /:shape       => triangle
+        /:state       => liquid
+
+    Variant green-square-gas-aluminum-anodic:    /run
+        /:coating     => anodic
+        /:color       => green
+        /:material    => aluminum
+        /:shape       => square
+        /:state       => gas
+
+    Variant gold-circle-solid-leather-anodic:    /run
+        /:coating     => anodic
+        /:color       => gold
+        /:material    => leather
+        /:shape       => circle
+        /:state       => solid
+
+    Variant green-triangle-solid-plastic-cathodic:    /run
+        /:coating     => cathodic
+        /:color       => green
+        /:material    => plastic
+        /:shape       => triangle
+        /:state       => solid
+
+    Variant gold-square-liquid-plastic-cathodic:    /run
+        /:coating     => cathodic
+        /:color       => gold
+        /:material    => plastic
+        /:shape       => square
+        /:state       => liquid
+
+    Variant black-triangle-gas-leather-anodic:    /run
+        /:coating     => anodic
+        /:color       => black
+        /:material    => leather
+        /:shape       => triangle
+        /:state       => gas
+
+    Variant black-circle-liquid-plastic-anodic:    /run
+        /:coating     => anodic
+        /:color       => black
+        /:material    => plastic
+        /:shape       => circle
+        /:state       => liquid
+
+    Variant gold-triangle-gas-aluminum-anodic:    /run
+        /:coating     => anodic
+        /:color       => gold
+        /:material    => aluminum
+        /:shape       => triangle
+        /:state       => gas
+
+    Variant green-circle-liquid-leather-anodic:    /run
+        /:coating     => anodic
+        /:color       => green
+        /:material    => leather
+        /:shape       => circle
+        /:state       => liquid
+
+    Variant red-square-solid-aluminum-anodic:    /run
+        /:coating     => anodic
+        /:color       => red
+        /:material    => aluminum
+        /:shape       => square
+        /:state       => solid
+
+    Variant black-triangle-liquid-aluminum-anodic:    /run
+        /:coating     => anodic
+        /:color       => black
+        /:material    => aluminum
+        /:shape       => triangle
+        /:state       => liquid
+
+    Variant red-square-gas-plastic-anodic:    /run
+        /:coating     => anodic
+        /:color       => red
+        /:material    => plastic
+        /:shape       => square
+        /:state       => gas
+
+    Variant red-square-liquid-leather-cathodic:    /run
+        /:coating     => cathodic
+        /:color       => red
+        /:material    => leather
+        /:shape       => square
+        /:state       => liquid
+
+To execute tests with those combinations use::
+
+    $ avocado run passtest.py --cit-parameter-file examples/varianter_cit/params.ini
+    JOB ID     : 6abd9e9f1ff9ed33a353ca8f3ef845cd4cc404a5
+    JOB LOG    : $HOME/avocado/job-results/job-2018-07-23T08.46-6abd9e9/job.log
+     (01/15) passtest.py:PassTest.test;gold-circle-gas-plastic-cathodic: PASS (0.06 s)
+     (02/15) passtest.py:PassTest.test;green-square-solid-plastic-anodic: PASS (0.02 s)
+     (03/15) passtest.py:PassTest.test;black-triangle-liquid-aluminum-anodic: PASS (0.02 s)
+     (04/15) passtest.py:PassTest.test;red-triangle-solid-leather-cathodic: PASS (0.02 s)
+     (05/15) passtest.py:PassTest.test;red-square-liquid-aluminum-cathodic: PASS (0.02 s)
+     (06/15) passtest.py:PassTest.test;green-circle-gas-leather-anodic: PASS (0.02 s)
+     (07/15) passtest.py:PassTest.test;gold-circle-solid-aluminum-anodic: PASS (0.02 s)
+     (08/15) passtest.py:PassTest.test;black-square-gas-leather-cathodic: PASS (0.02 s)
+     (09/15) passtest.py:PassTest.test;red-circle-liquid-plastic-anodic: PASS (0.02 s)
+     (10/15) passtest.py:PassTest.test;green-triangle-gas-aluminum-cathodic: PASS (0.02 s)
+     (11/15) passtest.py:PassTest.test;gold-triangle-liquid-leather-cathodic: PASS (0.02 s)
+     (12/15) passtest.py:PassTest.test;black-circle-solid-plastic-anodic: PASS (0.02 s)
+     (13/15) passtest.py:PassTest.test;red-triangle-gas-plastic-cathodic: PASS (0.02 s)
+     (14/15) passtest.py:PassTest.test;gold-square-liquid-leather-anodic: PASS (0.02 s)
+     (15/15) passtest.py:PassTest.test;green-circle-liquid-aluminum-anodic: PASS (0.02 s)
+    RESULTS    : PASS 15 | ERROR 0 | FAIL 0 | SKIP 0 | WARN 0 | INTERRUPT 0 | CANCEL 0
+    JOB TIME   : 1.21 s
+    JOB HTML   : $HOME/avocado/job-results/job-2018-07-23T08.46-6abd9e9/results.html

examples/varianter_cit/params.ini

+[parameters]
+color: black, gold, red, green
+shape: square, triangle, circle
+state: liquid, solid, gas
+material: leather, plastic, aluminum
+coating: anodic, cathodic

optional_plugins/varianter_cit/MANIFEST.in

+include VERSION

optional_plugins/varianter_cit/VERSION

+63.0

optional_plugins/varianter_cit/avocado_varianter_cit/__init__.py

+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See LICENSE for more details.
+# Authors: Amador Pahim <amador@pahim.org>
+#          Bestoun S. Ahmed <bestoon82@gmail.com>
+#          Cleber Rosa <crosa@redhat.com>
+
+import copy
+import itertools
+import os
+import random
+import sys
+
+from six import iteritems
+from six.moves import configparser
+from six.moves import zip
+
+from avocado.core import exit_codes
+from avocado.core.output import LOG_UI
+from avocado.core.plugin_interfaces import CLI
+from avocado.core.plugin_interfaces import Varianter
+from avocado.core.tree import TreeNode
+
+
+class VarianterCitCLI(CLI):
+
+    """
+    CIT Varianter options
+    """
+
+    name = 'cit'
+    description = "CIT Varianter options for the 'run' subcommand"
+
+    def configure(self, parser):
+
+        for name in ("run", "variants"):
+            subparser = parser.subcommands.choices.get(name, None)
+            if subparser is None:
+                continue
+            cit = subparser.add_argument_group('CIT varianter options')
+            cit.add_argument('--cit-parameter-file', metavar='PATH',
+                             help="Paths to a parameter file")
+            cit.add_argument('--cit-parameter-path', metavar='PATH',
+                             default='/run',
+                             help=('Default path for parameters generated '
+                                   'on the CIT variants'))
+            cit.add_argument('--cit-order-of-combinations',
+                             metavar='ORDER', type=int, default=2,
+                             help=("Order of combinations. Defaults to "
+                                   "%(default)s, maximum number is specific "
+                                   "to parameter file content"))
+
+    def run(self, args):
+        pass
+
+
+class VarianterCit(Varianter):
+
+    """
+    Processes the parameters file into variants
+    """
+
+    name = 'cit'
+    description = "CIT Varianter"
+
+    def initialize(self, args):
+        self.variants = None
+
+        cit_parameter_file = getattr(args, "cit_parameter_file", None)
+        if cit_parameter_file is None:
+            return
+        else:
+            cit_parameter_file = os.path.expanduser(cit_parameter_file)
+            if not os.access(cit_parameter_file, os.R_OK):
+                LOG_UI.error("parameter file '%s' could not be found or "
+                             "is not readable", cit_parameter_file)
+                self.error_exit(args)
+
+        self.parameter_path = getattr(args, "cit_parameter_path")
+        config = configparser.ConfigParser()
+        try:
+            config.read(cit_parameter_file)
+        except Exception as details:
+            LOG_UI.error("Cannot parse parameter file: %s", details)
+            self.error_exit(args)
+
+        parameters = [(key, value.split(', '))
+                      for key, value in config.items('parameters')]
+        order = args.cit_order_of_combinations
+        cit = Cit(parameters, order)
+        self.headers, self.variants = cit.combine()
+
+    @staticmethod
+    def error_exit(args):
+        if args.subcommand == 'run':
+            sys.exit(exit_codes.AVOCADO_JOB_FAIL)
+        else:
+            sys.exit(exit_codes.AVOCADO_FAIL)
+
+    def __iter__(self):
+        if self.variants is None:
+            return
+
+        variant_ids = []
+        for variant in self.variants:
+            variant_ids.append("-".join([variant.get(key)
+                                         for key in self.headers]))
+
+        for vid, variant in zip(variant_ids, self.variants):
+            yield {"variant_id": vid,
+                   "variant": TreeNode('', variant),
+                   "paths": self.parameter_path}
+
+    def __len__(self):
+        return sum(1 for _ in self.variants) if self.variants else 0
+
+    def update_defaults(self, defaults):
+        pass
+
+    def to_str(self, summary, variants, **kwargs):
+        """
+        Return human readable representation
+
+        The summary/variants accepts verbosity where 0 means silent and
+        maximum is up to the plugin.
+
+        :param summary: How verbose summary to output (int)
+        :param variants: How verbose list of variants to output (int)
+        :param kwargs: Other free-form arguments
+        :rtype: str
+        """
+        if not self.variants:
+            return ""
+
+        out = []
+        verbose = variants > 1
+        out.append("CIT Variants (%i):" % len(self))
+        for variant in self:
+            out.append('%sVariant %s:    %s' % ('\n' if verbose else '',
+                                                variant["variant_id"],
+                                                self.parameter_path))
+            if not verbose:
+                continue
+            env = set()
+            for node in variant["variant"]:
+                for key, value in iteritems(node.environment):
+                    origin = node.environment.origin[key].path
+                    env.add(("%s:%s" % (origin, key), str(value)))
+            if not env:
+                return out
+            fmt = '    %%-%ds => %%s' % max([len(_[0]) for _ in env])
+            for record in sorted(env):
+                out.append(fmt % record)
+
+        return "\n".join(out)
+
+
+class Cit(object):
+
+    MATRIX_ROW_SIZE = 20
+    MAX_ITERATIONS = 15
+
+    def __init__(self, parameters, order):
+        # Parameters come as ('key', ['value1', 'value2', 'value3'])
+        self.parameters = parameters
+        # Length (number of values) for each parameter
+        self.parameters_length = [len(param[1]) for param in self.parameters]
+        # Order of combinations
+        self.order = min(order, len(parameters))
+        self.hash_table = {}
+
+    def combine(self):
+        """
+        Computes the combination of parameters
+
+        :returns: headers (list of parameters keys) and combinations (list of
+                  dictionaries. Each dictionary represents a combination of
+                  parameters.
+        :rtype: tuple
+        """
+        self.create_interaction_hash_table()
+        final_list = self.create_final_list()
+
+        headers = [item[0] for item in self.parameters]
+        result = [[self.parameters[i][1][combination[i]]
+                   for i in range(len(combination))]
+                  for combination in final_list]
+        combinations = []
+        for combination in result:
+            combinations.append(dict(zip(headers, combination)))
+
+        return headers, combinations
+
+    def create_interaction_hash_table(self):
+        for c in itertools.combinations(range(len(self.parameters_length)), self.order):
+            self.hash_table[c] = self.get_iteration(c)
+
+    def create_final_list(self):
+        final_list = []
+        while len(self.hash_table) != 0:
+            iterations = 0
+            previous_test_case = []
+            previous_remove_list = {}
+            previous_weight = 0
+
+            while iterations < self.MAX_ITERATIONS:
+                max_width = len(self.hash_table)
+                matrix = self.create_random_matrix()
+                remove_list = {}
+                for i in matrix:
+                    width = self.get_weight(i, remove_list)
+                    if width == 0 or width <= previous_weight:
+                        remove_list.clear()
+                        continue
+                    elif width == max_width:
+                        final_list.append(i)
+                        self.remove_from_hash_table(remove_list)
+                        previous_test_case = []
+                        previous_remove_list = {}
+                        continue
+                    elif width > previous_weight:
+                        previous_weight = width
+                        previous_test_case = i
+                        previous_remove_list = dict(remove_list)
+                        remove_list.clear()
+                iterations += 1
+            if len(previous_test_case) != 0:
+                previous_remove_list = self.neighborhood_search(
+                    previous_test_case, previous_weight,
+                    max_width, previous_remove_list)
+                final_list.append(previous_test_case)
+                self.remove_from_hash_table(previous_remove_list)
+        return final_list
+
+    def get_iteration(self, parameter_combination):
+        parameters_array = []
+        for c in parameter_combination:
+            array = range(self.parameters_length[c])
+            parameters_array.append(array)
+        iterations = {}
+        for i in itertools.product(*parameters_array):
+            iterations[i] = tuple(i)
+        return iterations
+
+    def get_weight(self, test_case, remove_list):
+        weight = 0
+        for i in self.hash_table:
+            iteration = tuple(test_case[j] for j in i)
+            try:
+                value = self.hash_table[i][iteration]
+                weight += 1
+                remove_list[i] = value
+            except KeyError:
+                continue
+
+        return weight
+
+    def remove_from_hash_table(self, remove_list):
+        for i in remove_list:
+            del self.hash_table[i][remove_list[i]]
+            if len(self.hash_table[i]) == 0:
+                self.hash_table.pop(i)
+        remove_list.clear()
+
+    def create_random_matrix(self):
+        matrix = []
+        for _ in range(self.MATRIX_ROW_SIZE):
+            row = []
+            for j in self.parameters_length:
+                row.append(random.randint(0, j-1))
+            matrix.append(row)
+        return matrix
+
+    def neighborhood_search(self, test_case, width, max_width, remove_list):
+        neighborhood = list(test_case)
+        neighborhood_remove_list = {}
+        for i in range(len(test_case)):
+            # neighborhood +1
+            if (neighborhood[i] + 1) == self.parameters_length[i]:
+                neighborhood[i] = 0
+            else:
+                neighborhood[i] += 1
+            neighborhood_width = self.get_weight(neighborhood, neighborhood_remove_list)
+            if neighborhood_width > width:
+                width = neighborhood_width
+                remove_list = copy.deepcopy(neighborhood_remove_list)
+                del test_case[:]
+                for j in neighborhood:
+                    test_case.append(j)
+                if neighborhood_width == max_width:
+                    return remove_list
+            if neighborhood[i] == 0:
+                neighborhood[i] = self.parameters_length[i] - 1
+            else:
+                neighborhood[i] -= 1
+
+            # neighborhood -1
+            if neighborhood[i] == 0:
+                neighborhood[i] = self.parameters_length[i] - 1
+            else:
+                neighborhood[i] -= 1
+            neighborhood_width = self.get_weight(neighborhood, neighborhood_remove_list)
+            if neighborhood_width > width:
+                width = neighborhood_width
+                remove_list = copy.deepcopy(neighborhood_remove_list)
+                del test_case[:]
+                for j in neighborhood:
+                    test_case.append(j)
+                if neighborhood_width == max_width:
+                    return remove_list
+            if (neighborhood[i] + 1) == self.parameters_length[i]:
+                neighborhood[i] = 0
+            else:
+                neighborhood[i] += 1
+        return remove_list

optional_plugins/varianter_cit/setup.py

+#!/bin/env python
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See LICENSE for more details.
+# Authors: Amador Pahim <amador@pahim.org>
+#          Bestoun S. Ahmed <bestoon82@gmail.com>
+#          Cleber Rosa <crosa@redhat.com>
+
+
+from setuptools import setup, find_packages
+
+
+setup(name='avocado-framework-plugin-varianter-cit',
+      description='Varianter with combinatorial capabilities',
+      version=open("VERSION", "r").read().strip(),
+      author='Avocado Developers',
+      author_email='avocado-devel@redhat.com',
+      url='http://avocado-framework.github.io/',
+      packages=find_packages(),
+      include_package_data=True,
+      install_requires=['avocado-framework', ],
+      test_suite='tests',
+      entry_points={
+          'avocado.plugins.cli': [
+              'varianter_cit = avocado_varianter_cit:VarianterCitCLI',
+          ],
+          "avocado.plugins.varianter": [
+              "varianter_cit = avocado_varianter_cit:VarianterCit",
+          ]}
+      )

optional_plugins/varianter_cit/tests/test_functional.py

+import os
+import unittest
+
+from avocado.utils import process
+
+
+basedir = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', '..')
+basedir = os.path.abspath(basedir)
+
+AVOCADO = os.environ.get("UNITTEST_AVOCADO_CMD", "./scripts/avocado")
+
+
+class Variants(unittest.TestCase):
+
+    def test_max_variants(self):
+        os.chdir(basedir)
+        cmd_line = (
+            '{0} variants --cit-order-of-combinations=5 '
+            '--cit-parameter-file examples/varianter_cit/params.ini'
+        ).format(AVOCADO)
+        os.chdir(basedir)
+        result = process.run(cmd_line)
+        lines = result.stdout.splitlines()
+        self.assertEqual(b'CIT Variants (216):', lines[0])
+        self.assertEqual(217, len(lines))
+
+
+if __name__ == '__main__':
+    unittest.main()

optional_plugins/varianter_cit/tests/test_unit.py

+import unittest
+
+from avocado_varianter_cit import Cit
+
+
+class CitTest(unittest.TestCase):
+
+    PARAMS = [('key1', ['x1', 'x2', 'x3']), ('key2', ['y1', 'y2', 'y3']),
+              ('key3', ['z1', 'z2', 'z3']), ('key4', ['w1', 'w2', 'w3'])]
+
+    def test_orders(self):
+        for i in range(len(self.PARAMS)):
+            cit = Cit(self.PARAMS, i+1)
+            headers, _ = cit.combine()
+            # headers should always be the same, no matter the order
+            # of combinations
+            self.assertEqual(headers, ['key1', 'key2', 'key3', 'key4'])
+
+    def test_number_of_combinations(self):
+        # the algorithm doesn't allow us to know beforehand, for a
+        # reason, the precise number of combinations that will be
+        # computed.  still, we can check for the mininum number of
+        # combinations that should be produced
+        cit = Cit(self.PARAMS, 1)
+        self.assertEqual(len(cit.combine()[1]), 3)
+        cit = Cit(self.PARAMS, 2)
+        self.assertGreaterEqual(len(cit.combine()[1]), 9)
+        cit = Cit(self.PARAMS, 3)
+        self.assertGreaterEqual(len(cit.combine()[1]), 27)
+
+    def test_max_order(self):
+        # test that with a order equal or larger than the number of
+        # keys, we have a predictable number of combinations
+        cit = Cit(self.PARAMS, 4)
+        self.assertEqual(len(cit.combine()[1]), 81)
+        cit = Cit(self.PARAMS, 10)
+        self.assertEqual(len(cit.combine()[1]), 81)
+
+
+if __name__ == '__main__':
+    unittest.main()

selftests/run

@@ -38,7 +38,9 @@ def test_suite():
                ('avocado-framework-plugin-runner-remote',
                 'runner_remote'),
                ('avocado-framework-plugin-runner-vm',
-                'runner_vm'))
+                'runner_vm'),
+               ('avocado-framework-plugin-varianter-cit',
+                'varianter_cit'))
     for plugin_name, plugin_dir in plugins:
         if plugin_available(plugin_name):
             path = os.path.join(basedir, 'optional_plugins',