[JOB API] Test features/options using Python dictionary variants.

This is the initial support to test the Avocado features/options
using the Job API.
Signed-off-by: NWillian Rampazzo <willianr@redhat.com>

selftests/job_api/test_features.py

+#!/usr/bin/env python3
+
+import os
+
+from avocado import Test
+from avocado.core import exit_codes
+from avocado.core.job import Job
+from avocado.core.suite import TestSuite
+
+BOOLEAN_ENABLED = [True, 'true', 'on', 1]
+BOOLEAN_DISABLED = [False, 'false', 'off', 0]
+
+
+class JobAPIFeaturesTest(Test):
+
+    def check_directory_exists(self, path):
+        """Check if a directory exists"""
+        self.assertTrue(os.path.isdir(path))
+
+    def check_exit_code(self, exit_code):
+        """Check if job ended with success."""
+        expected_exit_code = self.params.get('exit_code',
+                                             default=exit_codes.AVOCADO_ALL_OK)
+        self.assertEqual(expected_exit_code, exit_code)
+
+    def check_file_exists(self, file_path):
+        """Check if a file exists or not depending on the `assert_func`."""
+        assert_func = self.get_assert_function()
+        assert_func(os.path.exists(file_path))
+
+    def check_file_content(self, file_path):
+        """Check if `content` exists or not in a file."""
+        content = self.params.get('content')
+        assert_func = self.get_assert_function()
+        assert_func(self.file_has_content(file_path, content))
+
+    def create_config(self, value):
+        """Creates the Job config."""
+        reference = self.params.get('reference', default=['/bin/true'])
+        config = {'core.show': ['none'],
+                  'run.results_dir': self.workdir,
+                  'run.references': reference}
+        namespace = self.params.get('namespace')
+        config[namespace] = value
+
+        return config
+
+    @staticmethod
+    def file_has_content(file_path, content):
+        """Check if a file has `content`."""
+        if os.path.isfile(file_path):
+            with open(file_path, "r") as f:
+                source_content = f.read()
+            if content in source_content:
+                return True
+        return False
+
+    def get_assert_function(self):
+        """Return an assert function depending on the assert passed"""
+        assert_option = self.params.get('assert')
+        if assert_option:
+            return self.assertTrue
+        return self.assertFalse
+
+    @property
+    def latest_workdir(self):
+        return os.path.join(self.workdir, 'latest')
+
+    def run_job(self):
+        """Run a Job"""
+        config = self.create_config(self.params.get('value'))
+
+        suite = TestSuite.from_config(config)
+
+        # run the job
+        with Job(config, [suite]) as j:
+            result = j.run()
+
+        return result
+
+    @property
+    def workdir_file_path(self):
+        file_name = self.params.get('file')
+        return os.path.join(self.latest_workdir, file_name)
+
+    def test_check_directory_exists(self):
+        """Test to check if a directory was created."""
+        config = self.create_config(self.params.get('value'))
+
+        suite = TestSuite.from_config(config)
+
+        # run the job
+        with Job(config, [suite]) as j:
+            result = j.run()
+            tmpdir = os.path.basename(j.tmpdir)
+
+        # Asserts
+        self.check_exit_code(result)
+        self.check_directory_exists(os.path.join(self.latest_workdir, tmpdir))
+
+    def test_check_file_content(self):
+        """Test to check if a file has the desired content."""
+        result = self.run_job()
+
+        # Asserts
+        self.check_exit_code(result)
+        self.check_file_content(self.workdir_file_path)
+
+    def test_check_file_exists(self):
+        """Test to check if a file was created."""
+        result = self.run_job()
+
+        # Asserts
+        self.check_exit_code(result)
+        self.check_file_exists(self.workdir_file_path)
+
+    def test_check_output_file(self):
+        """Test to check if the file passed as parameter was created."""
+        config = self.create_config(self.workdir_file_path)
+
+        suite = TestSuite.from_config(config)
+
+        # run the job
+        with Job(config, [suite]) as j:
+            result = j.run()
+
+        # Asserts
+        self.check_exit_code(result)
+        self.check_file_exists(self.workdir_file_path)
+
+if __name__ == '__main__':
+
+    test_class = 'JobAPIFeaturesTest'
+    suites = []
+
+    # ========================================================================
+    # Test if a directory was created
+    # ========================================================================
+    check_directory_exists = ('%s:%s.test_check_directory_exists'
+                              % (__file__, test_class))
+    config_check_directory_exists = (
+        {'run.references': [check_directory_exists],
+         'run.dict_variants': [
+
+             {'namespace': 'run.keep_tmp',
+              'value': True},
+
+         ]})
+
+    suites.append(TestSuite.from_config(config_check_directory_exists))
+
+    # ========================================================================
+    # Test the content of a file
+    # ========================================================================
+    check_file_content = ('%s:%s.test_check_file_content'
+                          % (__file__, test_class))
+    config_check_file_content = (
+        {'run.references': [check_file_content],
+         'run.dict_variants': [
+
+             # finding the correct 'content' here is trick because any
+             # simple string is added to the variant file name and is
+             # found in the log file.
+             # Using DEBUG| makes the variant name have DEBUG_, working
+             # fine here.
+             {'namespace': 'job.output.loglevel',
+              'value': 'INFO',
+              'file': 'job.log',
+              'content': 'DEBUG| Test metadata',
+              'assert': False},
+
+             {'namespace': 'job.run.result.tap.include_logs',
+              'value': True,
+              'file': 'results.tap',
+              'content': "Command '/bin/true' finished with 0",
+              'assert': True},
+
+             {'namespace': 'job.run.result.tap.include_logs',
+              'value': False,
+              'file': 'results.tap',
+              'content': "Command '/bin/true' finished with 0",
+              'assert': False},
+
+             {'namespace': 'job.run.result.xunit.job_name',
+              'value': 'foo',
+              'file': 'results.xml',
+              'content': 'name="foo"',
+              'assert': True},
+
+             {'namespace': 'job.run.result.xunit.max_test_log_chars',
+              'value': 1,
+              'file': 'results.xml',
+              'content': '--[ CUT DUE TO XML PER TEST LIMIT ]--',
+              'assert': True,
+              'reference': ['/bin/false'],
+              'exit_code': 1},
+
+             {'namespace': 'run.failfast',
+              'value': True,
+              'file': 'results.json',
+              'content': '"skip": 1',
+              'assert': True,
+              'reference': ['/bin/false', '/bin/true'],
+              'exit_code': 9},
+
+             {'namespace': 'run.ignore_missing_references',
+              'value': 'on',
+              'file': 'results.json',
+              'content': '"pass": 1',
+              'assert': True,
+              'reference': ['/bin/true', 'foo'],
+              'exit_code': 0},
+
+         ]})
+
+    suites.append(TestSuite.from_config(config_check_file_content))
+
+    # ========================================================================
+    # Test if the result file was created
+    # ========================================================================
+    check_file_exists = ('%s:%s.test_check_file_exists'
+                         % (__file__, test_class))
+    config_check_file_exists = (
+        {'run.references': [check_file_exists],
+         'run.dict_variants': [
+
+             {'namespace': 'job.run.result.html.enabled',
+              'value': 'on',
+              'file': 'results.html',
+              'assert': True},
+
+             {'namespace': 'job.run.result.html.enabled',
+              'value': 'off',
+              'file': 'results.html',
+              'assert': False},
+
+             {'namespace': 'job.run.result.json.enabled',
+              'value': 'on',
+              'file': 'results.json',
+              'assert': True},
+
+             {'namespace': 'job.run.result.json.enabled',
+              'value': 'off',
+              'file': 'results.json',
+              'assert': False},
+
+             {'namespace': 'job.run.result.tap.enabled',
+              'value': 'on',
+              'file': 'results.tap',
+              'assert': True},
+
+             {'namespace': 'job.run.result.tap.enabled',
+              'value': 'off',
+              'file': 'results.tap',
+              'assert': False},
+
+             {'namespace': 'job.run.result.xunit.enabled',
+              'value': 'on',
+              'file': 'results.xml',
+              'assert': True},
+
+             {'namespace': 'job.run.result.xunit.enabled',
+              'value': 'off',
+              'file': 'results.xml',
+              'assert': False},
+
+             {'namespace': 'run.dry_run.enabled',
+              'value': True,
+              'file': 'job.log',
+              'assert': False},
+
+             {'namespace': 'run.dry_run.no_cleanup',
+              'value': True,
+              'file': 'job.log',
+              'assert': True},
+
+         ]})
+
+    suites.append(TestSuite.from_config(config_check_file_exists))
+
+    # ========================================================================
+    # Test if a file was created
+    # ========================================================================
+    check_output_file = ('%s:%s.test_check_output_file'
+                         % (__file__, test_class))
+    config_check_output_file = (
+        {'run.references': [check_output_file],
+         'run.dict_variants': [
+
+             {'namespace': 'job.run.result.html.output',
+              'file': 'custom.html',
+              'assert': True},
+
+             {'namespace': 'job.run.result.json.output',
+              'file': 'custom.json',
+              'assert': True},
+
+             # https://github.com/avocado-framework/avocado/issues/4034
+             {'namespace': 'job.run.result.tap.output',
+              'file': 'custom.tap',
+              'assert': True},
+
+             {'namespace': 'job.run.result.xunit.output',
+              'file': 'custom.xml',
+              'assert': True},
+
+         ]})
+
+    suites.append(TestSuite.from_config(config_check_output_file))
+
+    # ========================================================================
+    # Job execution
+    # ========================================================================
+    config = {'core.show': ['app']}
+    with Job(config, suites) as j:
+        j.run()