test_basic.py 51.8 KB
Newer Older
1
# This Python file uses the following encoding: utf-8
2 3
import aexpect
import glob
4
import json
5
import os
6
import re
7
import shutil
8
import signal
9
import sys
10
import tempfile
11
import time
12
import xml.dom.minidom
13
import zipfile
14
import unittest
15
import psutil
16
import pkg_resources
17

18 19 20
from lxml import etree
from StringIO import StringIO

21
from avocado.core import exit_codes
22
from avocado.utils import astring
23 24
from avocado.utils import process
from avocado.utils import script
25
from avocado.utils import path as utils_path
26

27
basedir = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..')
28 29
basedir = os.path.abspath(basedir)

30 31
AVOCADO = os.environ.get("UNITTEST_AVOCADO_CMD", "./scripts/avocado")

32 33 34 35
PASS_SCRIPT_CONTENTS = """#!/bin/sh
true
"""

C
Cleber Rosa 已提交
36 37
PASS_SHELL_CONTENTS = "exit 0"

38 39 40 41
FAIL_SCRIPT_CONTENTS = """#!/bin/sh
false
"""

C
Cleber Rosa 已提交
42 43
FAIL_SHELL_CONTENTS = "exit 1"

44 45 46 47 48 49 50 51 52 53 54 55 56 57
HELLO_LIB_CONTENTS = """
def hello():
    return 'Hello world'
"""

LOCAL_IMPORT_TEST_CONTENTS = '''
from avocado import Test
from mylib import hello

class LocalImportTest(Test):
    def test(self):
        self.log.info(hello())
'''

58 59 60 61 62 63
UNSUPPORTED_STATUS_TEST_CONTENTS = '''
from avocado import Test

class FakeStatusTest(Test):
    def run_avocado(self):
        super(FakeStatusTest, self).run_avocado()
64 65
        # Please do NOT ever use this, it's for unittesting only.
        self._Test__status = 'not supported'
66 67 68 69 70

    def test(self):
        pass
'''

71 72 73 74 75 76 77 78 79 80 81
INVALID_PYTHON_TEST = '''
from avocado import Test

class MyTest(Test):

    non_existing_variable_causing_crash

    def test_my_name(self):
        pass
'''

82

83 84 85 86 87 88 89 90 91 92
REPORTS_STATUS_AND_HANG = '''
from avocado import Test
import time

class MyTest(Test):
    def test(self):
         self.runner_queue.put({"running": False})
         time.sleep(60)
'''

93

94 95 96 97 98 99 100 101 102 103 104
DIE_WITHOUT_REPORTING_STATUS = '''
from avocado import Test
import os
import signal

class MyTest(Test):
    def test(self):
         os.kill(os.getpid(), signal.SIGKILL)
'''


A
Amador Pahim 已提交
105
def probe_binary(binary):
106
    try:
A
Amador Pahim 已提交
107
        return utils_path.find_command(binary)
108
    except utils_path.CmdNotFoundError:
A
Amador Pahim 已提交
109 110
        return None

L
Lukáš Doktor 已提交
111

112
TRUE_CMD = probe_binary('true')
A
Amador Pahim 已提交
113
CC_BINARY = probe_binary('cc')
114 115 116 117 118 119 120

# On macOS, the default GNU coreutils installation (brew)
# installs the gnu utility versions with a g prefix. It still has the
# BSD versions of the core utilities installed on their expected paths
# but their behavior and flags are in most cases different.
GNU_ECHO_BINARY = probe_binary('echo')
if GNU_ECHO_BINARY is not None:
121 122 123 124
    if probe_binary('man') is not None:
        echo_manpage = process.run('man %s' % os.path.basename(GNU_ECHO_BINARY)).stdout
        if '-e' not in echo_manpage:
            GNU_ECHO_BINARY = probe_binary('gecho')
A
Amador Pahim 已提交
125 126
READ_BINARY = probe_binary('read')
SLEEP_BINARY = probe_binary('sleep')
127 128


129 130
class RunnerOperationTest(unittest.TestCase):

131
    def setUp(self):
132
        self.tmpdir = tempfile.mkdtemp(prefix='avocado_' + __name__)
133

134
    def test_show_version(self):
135
        result = process.run('%s -v' % AVOCADO, ignore_status=True)
136
        self.assertEqual(result.exit_status, 0)
C
Cleber Rosa 已提交
137 138 139
        self.assertTrue(re.match(r"^Avocado \d+\.\d+$", result.stderr),
                        "Version string does not match 'Avocado \\d\\.\\d:'\n"
                        "%r" % (result.stderr))
140

141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163
    def test_alternate_config_datadir(self):
        """
        Uses the "--config" flag to check custom configuration is applied

        Even on the more complex data_dir module, which adds extra checks
        to what is set on the plain settings module.
        """
        base_dir = os.path.join(self.tmpdir, 'datadir_base')
        os.mkdir(base_dir)
        mapping = {'base_dir': base_dir,
                   'test_dir': os.path.join(base_dir, 'test'),
                   'data_dir': os.path.join(base_dir, 'data'),
                   'logs_dir': os.path.join(base_dir, 'logs')}
        config = '[datadir.paths]'
        for key, value in mapping.iteritems():
            if not os.path.isdir(value):
                os.mkdir(value)
            config += "%s = %s\n" % (key, value)
        fd, config_file = tempfile.mkstemp(dir=self.tmpdir)
        os.write(fd, config)
        os.close(fd)

        os.chdir(basedir)
164
        cmd = '%s --config %s config --datadir' % (AVOCADO, config_file)
165 166 167 168 169 170 171 172 173 174
        result = process.run(cmd)
        output = result.stdout
        expected_rc = exit_codes.AVOCADO_ALL_OK
        self.assertEqual(result.exit_status, expected_rc,
                         "Avocado did not return rc %d:\n%s" %
                         (expected_rc, result))
        self.assertIn('    base     ' + mapping['base_dir'], result.stdout)
        self.assertIn('    data     ' + mapping['data_dir'], result.stdout)
        self.assertIn('    logs     ' + mapping['logs_dir'], result.stdout)

175 176
    def test_runner_all_ok(self):
        os.chdir(basedir)
177 178
        cmd_line = ('%s run --sysinfo=off --job-results-dir %s '
                    'passtest.py passtest.py' % (AVOCADO, self.tmpdir))
179 180
        process.run(cmd_line)

181 182
    def test_runner_failfast(self):
        os.chdir(basedir)
183 184 185
        cmd_line = ('%s run --sysinfo=off --job-results-dir %s '
                    'passtest.py failtest.py passtest.py --failfast on'
                    % (AVOCADO, self.tmpdir))
186 187 188 189 190 191 192
        result = process.run(cmd_line, ignore_status=True)
        self.assertIn('Interrupting job (failfast).', result.stdout)
        self.assertIn('PASS 1 | ERROR 0 | FAIL 1 | SKIP 1', result.stdout)
        expected_rc = exit_codes.AVOCADO_TESTS_FAIL | exit_codes.AVOCADO_JOB_INTERRUPTED
        self.assertEqual(result.exit_status, expected_rc,
                         "Avocado did not return rc %d:\n%s" % (expected_rc, result))

A
Amador Pahim 已提交
193
    @unittest.skipIf(not CC_BINARY,
194
                     "C compiler is required by the underlying datadir.py test")
195 196
    def test_datadir_alias(self):
        os.chdir(basedir)
197 198
        cmd_line = ('%s run --sysinfo=off --job-results-dir %s '
                    'datadir.py' % (AVOCADO, self.tmpdir))
199 200 201 202 203
        process.run(cmd_line)

    def test_shell_alias(self):
        """ Tests that .sh files are also executable via alias """
        os.chdir(basedir)
204 205
        cmd_line = ('%s run --sysinfo=off --job-results-dir %s '
                    'env_variables.sh' % (AVOCADO, self.tmpdir))
206 207
        process.run(cmd_line)

A
Amador Pahim 已提交
208
    @unittest.skipIf(not CC_BINARY,
209
                     "C compiler is required by the underlying datadir.py test")
210 211
    def test_datadir_noalias(self):
        os.chdir(basedir)
212 213
        cmd_line = ('%s run --sysinfo=off --job-results-dir %s examples/tests/datadir.py '
                    'examples/tests/datadir.py' % (AVOCADO, self.tmpdir))
214 215
        process.run(cmd_line)

216 217
    def test_runner_noalias(self):
        os.chdir(basedir)
218 219
        cmd_line = ("%s run --sysinfo=off --job-results-dir %s examples/tests/passtest.py "
                    "examples/tests/passtest.py" % (AVOCADO, self.tmpdir))
220 221
        process.run(cmd_line)

222 223 224 225 226 227 228 229 230 231 232
    def test_runner_test_with_local_imports(self):
        mylib = script.TemporaryScript(
            'mylib.py',
            HELLO_LIB_CONTENTS,
            'avocado_simpletest_functional')
        mylib.save()
        mytest = script.Script(
            os.path.join(os.path.dirname(mylib.path), 'test_local_imports.py'),
            LOCAL_IMPORT_TEST_CONTENTS)
        os.chdir(basedir)
        mytest.save()
233 234
        cmd_line = ("%s run --sysinfo=off --job-results-dir %s "
                    "%s" % (AVOCADO, self.tmpdir, mytest))
235 236
        process.run(cmd_line)

237 238 239 240 241
    def test_unsupported_status(self):
        os.chdir(basedir)
        with script.TemporaryScript("fake_status.py",
                                    UNSUPPORTED_STATUS_TEST_CONTENTS,
                                    "avocado_unsupported_status") as tst:
242 243 244
            res = process.run("%s run --sysinfo=off --job-results-dir %s %s"
                              " --json -" % (AVOCADO, self.tmpdir, tst),
                              ignore_status=True)
245 246 247 248 249
            self.assertEqual(res.exit_status, exit_codes.AVOCADO_TESTS_FAIL)
            results = json.loads(res.stdout)
            self.assertEqual(results["tests"][0]["status"], "ERROR",
                             "%s != %s\n%s" % (results["tests"][0]["status"],
                                               "ERROR", res))
250
            self.assertIn("Runner error occurred: Test reports unsupported",
251 252
                          results["tests"][0]["fail_reason"])

253 254 255 256 257 258
    def test_hanged_test_with_status(self):
        """ Check that avocado handles hanged tests properly """
        os.chdir(basedir)
        with script.TemporaryScript("report_status_and_hang.py",
                                    REPORTS_STATUS_AND_HANG,
                                    "hanged_test_with_status") as tst:
259 260 261
            res = process.run("%s run --sysinfo=off --job-results-dir %s %s "
                              "--json -" % (AVOCADO, self.tmpdir, tst),
                              ignore_status=True)
262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277
            self.assertEqual(res.exit_status, exit_codes.AVOCADO_TESTS_FAIL)
            results = json.loads(res.stdout)
            self.assertEqual(results["tests"][0]["status"], "ERROR",
                             "%s != %s\n%s" % (results["tests"][0]["status"],
                                               "ERROR", res))
            self.assertIn("Test reported status but did not finish",
                          results["tests"][0]["fail_reason"])
            self.assertLess(res.duration, 40, "Test execution took too long, "
                            "which is likely because the hanged test was not "
                            "interrupted. Results:\n%s" % res)

    def test_no_status_reported(self):
        os.chdir(basedir)
        with script.TemporaryScript("die_without_reporting_status.py",
                                    DIE_WITHOUT_REPORTING_STATUS,
                                    "no_status_reported") as tst:
278 279 280
            res = process.run("%s run --sysinfo=off --job-results-dir %s %s "
                              "--json -" % (AVOCADO, self.tmpdir, tst),
                              ignore_status=True)
281 282 283 284 285 286 287 288
            self.assertEqual(res.exit_status, exit_codes.AVOCADO_TESTS_FAIL)
            results = json.loads(res.stdout)
            self.assertEqual(results["tests"][0]["status"], "ERROR",
                             "%s != %s\n%s" % (results["tests"][0]["status"],
                                               "ERROR", res))
            self.assertIn("Test died without reporting the status",
                          results["tests"][0]["fail_reason"])

289 290
    def test_runner_tests_fail(self):
        os.chdir(basedir)
291 292
        cmd_line = ('%s run --sysinfo=off --job-results-dir %s passtest.py '
                    'failtest.py passtest.py' % (AVOCADO, self.tmpdir))
293
        result = process.run(cmd_line, ignore_status=True)
294
        expected_rc = exit_codes.AVOCADO_TESTS_FAIL
295 296 297 298 299
        self.assertEqual(result.exit_status, expected_rc,
                         "Avocado did not return rc %d:\n%s" % (expected_rc, result))

    def test_runner_nonexistent_test(self):
        os.chdir(basedir)
300 301
        cmd_line = ('%s run --sysinfo=off --job-results-dir '
                    '%s bogustest' % (AVOCADO, self.tmpdir))
302
        result = process.run(cmd_line, ignore_status=True)
303 304
        expected_rc = exit_codes.AVOCADO_JOB_FAIL
        unexpected_rc = exit_codes.AVOCADO_FAIL
305 306 307 308 309
        self.assertNotEqual(result.exit_status, unexpected_rc,
                            "Avocado crashed (rc %d):\n%s" % (unexpected_rc, result))
        self.assertEqual(result.exit_status, expected_rc,
                         "Avocado did not return rc %d:\n%s" % (expected_rc, result))

310 311
    def test_runner_doublefail(self):
        os.chdir(basedir)
312 313
        cmd_line = ('%s run --sysinfo=off --job-results-dir %s '
                    '--xunit - doublefail.py' % (AVOCADO, self.tmpdir))
314 315
        result = process.run(cmd_line, ignore_status=True)
        output = result.stdout
316 317
        expected_rc = exit_codes.AVOCADO_TESTS_FAIL
        unexpected_rc = exit_codes.AVOCADO_FAIL
318 319 320 321
        self.assertNotEqual(result.exit_status, unexpected_rc,
                            "Avocado crashed (rc %d):\n%s" % (unexpected_rc, result))
        self.assertEqual(result.exit_status, expected_rc,
                         "Avocado did not return rc %d:\n%s" % (expected_rc, result))
322
        self.assertIn("TestError: Failing during tearDown. Yay!", output,
323
                      "Cleanup exception not printed to log output")
324
        self.assertIn("TestFail: This test is supposed to fail",
325
                      output,
326
                      "Test did not fail with action exception:\n%s" % output)
327

328 329
    def test_uncaught_exception(self):
        os.chdir(basedir)
330 331
        cmd_line = ("%s run --sysinfo=off --job-results-dir %s "
                    "--json - uncaught_exception.py" % (AVOCADO, self.tmpdir))
332
        result = process.run(cmd_line, ignore_status=True)
333
        expected_rc = exit_codes.AVOCADO_TESTS_FAIL
334 335 336 337 338
        self.assertEqual(result.exit_status, expected_rc,
                         "Avocado did not return rc %d:\n%s" % (expected_rc,
                                                                result))
        self.assertIn('"status": "ERROR"', result.stdout)

339
    def test_fail_on_exception(self):
340
        os.chdir(basedir)
341 342
        cmd_line = ("%s run --sysinfo=off --job-results-dir %s "
                    "--json - fail_on_exception.py" % (AVOCADO, self.tmpdir))
343
        result = process.run(cmd_line, ignore_status=True)
344
        expected_rc = exit_codes.AVOCADO_TESTS_FAIL
345 346 347 348 349
        self.assertEqual(result.exit_status, expected_rc,
                         "Avocado did not return rc %d:\n%s" % (expected_rc,
                                                                result))
        self.assertIn('"status": "FAIL"', result.stdout)

350 351
    def test_runner_timeout(self):
        os.chdir(basedir)
352 353
        cmd_line = ('%s run --sysinfo=off --job-results-dir %s '
                    '--xunit - timeouttest.py' % (AVOCADO, self.tmpdir))
354 355
        result = process.run(cmd_line, ignore_status=True)
        output = result.stdout
356
        expected_rc = exit_codes.AVOCADO_JOB_INTERRUPTED
357
        unexpected_rc = exit_codes.AVOCADO_FAIL
358 359 360 361
        self.assertNotEqual(result.exit_status, unexpected_rc,
                            "Avocado crashed (rc %d):\n%s" % (unexpected_rc, result))
        self.assertEqual(result.exit_status, expected_rc,
                         "Avocado did not return rc %d:\n%s" % (expected_rc, result))
362
        self.assertIn("Runner error occurred: Timeout reached", output,
363
                      "Timeout reached message not found in the output:\n%s" % output)
364 365
        # Ensure no test aborted error messages show up
        self.assertNotIn("TestAbortedError: Test aborted unexpectedly", output)
366

367 368 369
    @unittest.skipIf(os.environ.get("AVOCADO_CHECK_FULL") != "1",
                     "Skipping test that take a long time to run, are "
                     "resource intensive or time sensitve")
370 371
    def test_runner_abort(self):
        os.chdir(basedir)
372 373
        cmd_line = ('%s run --sysinfo=off --job-results-dir %s '
                    '--xunit - abort.py' % (AVOCADO, self.tmpdir))
374
        result = process.run(cmd_line, ignore_status=True)
375
        output = result.stdout
376
        excerpt = 'Test died without reporting the status.'
377 378
        expected_rc = exit_codes.AVOCADO_TESTS_FAIL
        unexpected_rc = exit_codes.AVOCADO_FAIL
379 380 381 382
        self.assertNotEqual(result.exit_status, unexpected_rc,
                            "Avocado crashed (rc %d):\n%s" % (unexpected_rc, result))
        self.assertEqual(result.exit_status, expected_rc,
                         "Avocado did not return rc %d:\n%s" % (expected_rc, result))
383
        self.assertIn(excerpt, output)
384

385 386
    def test_silent_output(self):
        os.chdir(basedir)
387 388
        cmd_line = ('%s --silent run --sysinfo=off --job-results-dir %s '
                    'passtest.py' % (AVOCADO, self.tmpdir))
389
        result = process.run(cmd_line, ignore_status=True)
390
        expected_rc = exit_codes.AVOCADO_ALL_OK
391 392
        expected_output = ''
        self.assertEqual(result.exit_status, expected_rc)
393
        self.assertEqual(result.stdout, expected_output)
394

395 396
    def test_empty_args_list(self):
        os.chdir(basedir)
397
        cmd_line = AVOCADO
398
        result = process.run(cmd_line, ignore_status=True)
399
        expected_rc = exit_codes.AVOCADO_FAIL
400
        expected_output = 'error: too few arguments'
401
        self.assertEqual(result.exit_status, expected_rc)
402
        self.assertIn(expected_output, result.stderr)
403

404 405
    def test_empty_test_list(self):
        os.chdir(basedir)
406 407
        cmd_line = '%s run --sysinfo=off --job-results-dir %s' % (AVOCADO,
                                                                  self.tmpdir)
408
        result = process.run(cmd_line, ignore_status=True)
409
        expected_rc = exit_codes.AVOCADO_JOB_FAIL
410 411
        expected_output = ('No test references provided nor any other '
                           'arguments resolved into tests')
412
        self.assertEqual(result.exit_status, expected_rc)
413
        self.assertIn(expected_output, result.stderr)
414

415 416
    def test_not_found(self):
        os.chdir(basedir)
417 418
        cmd_line = ('%s run --sysinfo=off --job-results-dir %s sbrubles'
                    % (AVOCADO, self.tmpdir))
419
        result = process.run(cmd_line, ignore_status=True)
420
        expected_rc = exit_codes.AVOCADO_JOB_FAIL
421
        self.assertEqual(result.exit_status, expected_rc)
422 423
        self.assertIn('Unable to resolve reference', result.stderr)
        self.assertNotIn('Unable to resolve reference', result.stdout)
424

425
    def test_invalid_unique_id(self):
426 427
        cmd_line = ('%s run --sysinfo=off --job-results-dir %s --force-job-id '
                    'foobar passtest.py' % (AVOCADO, self.tmpdir))
428
        result = process.run(cmd_line, ignore_status=True)
429
        self.assertNotEqual(result.exit_status, exit_codes.AVOCADO_ALL_OK)
430
        self.assertIn('needs to be a 40 digit hex', result.stderr)
431
        self.assertNotIn('needs to be a 40 digit hex', result.stdout)
432 433

    def test_valid_unique_id(self):
434
        cmd_line = ('%s run --job-results-dir %s --sysinfo=off '
435
                    '--force-job-id 975de258ac05ce5e490648dec4753657b7ccc7d1 '
436
                    'passtest.py' % (AVOCADO, self.tmpdir))
437
        result = process.run(cmd_line, ignore_status=True)
438
        self.assertEqual(result.exit_status, exit_codes.AVOCADO_ALL_OK)
439
        self.assertNotIn('needs to be a 40 digit hex', result.stderr)
440
        self.assertIn('PASS', result.stdout)
441

442
    def test_automatic_unique_id(self):
443 444
        cmd_line = ('%s run --job-results-dir %s --sysinfo=off '
                    'passtest.py --json -' % (AVOCADO, self.tmpdir))
445
        result = process.run(cmd_line, ignore_status=True)
446
        self.assertEqual(result.exit_status, exit_codes.AVOCADO_ALL_OK)
447 448 449 450
        r = json.loads(result.stdout)
        int(r['job_id'], 16)  # it's an hex number
        self.assertEqual(len(r['job_id']), 40)

451 452 453 454 455
    def test_early_latest_result(self):
        """
        Tests that the `latest` link to the latest job results is created early
        """
        os.chdir(basedir)
456 457
        cmd_line = ('%s run --sysinfo=off --job-results-dir %s '
                    'examples/tests/passtest.py' % (AVOCADO, self.tmpdir))
458 459 460 461 462 463
        avocado_process = process.SubProcess(cmd_line)
        avocado_process.start()
        link = os.path.join(self.tmpdir, 'latest')
        for trial in xrange(0, 50):
            time.sleep(0.1)
            if os.path.exists(link) and os.path.islink(link):
464
                avocado_process.wait()
465 466 467 468
                break
        self.assertTrue(os.path.exists(link))
        self.assertTrue(os.path.islink(link))

469 470
    def test_dry_run(self):
        os.chdir(basedir)
471
        cmd = ("%s run --sysinfo=off passtest.py failtest.py "
472
               "gendata.py --json - --mux-inject foo:1 bar:2 baz:3 foo:foo:a"
473
               " foo:bar:b foo:baz:c bar:bar:bar --dry-run" % AVOCADO)
474 475 476 477 478
        result = json.loads(process.run(cmd).stdout)
        debuglog = result['debuglog']
        log = open(debuglog, 'r').read()
        # Remove the result dir
        shutil.rmtree(os.path.dirname(os.path.dirname(debuglog)))
479
        self.assertIn(tempfile.gettempdir(), debuglog)   # Use tmp dir, not default location
480 481
        self.assertEqual(result['job_id'], u'0' * 40)
        # Check if all tests were skipped
482 483
        self.assertEqual(result['skip'], 4)
        for i in xrange(4):
484 485 486 487 488 489 490 491
            test = result['tests'][i]
            self.assertEqual(test['fail_reason'],
                             u'Test skipped due to --dry-run')
        # Check if all params are listed
        # The "/:bar ==> 2 is in the tree, but not in any leave so inaccessible
        # from test.
        for line in ("/:foo ==> 1", "/:baz ==> 3", "/foo:foo ==> a",
                     "/foo:bar ==> b", "/foo:baz ==> c", "/bar:bar ==> bar"):
492
            self.assertEqual(log.count(line), 4)
493

494 495 496 497
    def test_invalid_python(self):
        os.chdir(basedir)
        test = script.make_script(os.path.join(self.tmpdir, 'test.py'),
                                  INVALID_PYTHON_TEST)
498 499
        cmd_line = ('%s --show test run --sysinfo=off '
                    '--job-results-dir %s %s') % (AVOCADO, self.tmpdir, test)
500 501 502 503 504
        result = process.run(cmd_line, ignore_status=True)
        expected_rc = exit_codes.AVOCADO_TESTS_FAIL
        self.assertEqual(result.exit_status, expected_rc,
                         "Avocado did not return rc %d:\n%s" %
                         (expected_rc, result))
505 506
        self.assertIn('1-%s:MyTest.test_my_name -> TestError' % test,
                      result.stdout)
507

A
Amador Pahim 已提交
508
    @unittest.skipIf(not READ_BINARY, "read binary not available.")
L
Lukáš Doktor 已提交
509 510
    def test_read(self):
        os.chdir(basedir)
511
        cmd = "%s run --sysinfo=off --job-results-dir %%s %%s" % AVOCADO
512
        cmd %= (self.tmpdir, READ_BINARY)
513
        result = process.run(cmd, timeout=10, ignore_status=True)
L
Lukáš Doktor 已提交
514 515 516 517 518
        self.assertLess(result.duration, 8, "Duration longer than expected."
                        "\n%s" % result)
        self.assertEqual(result.exit_status, 1, "Expected exit status is 1\n%s"
                         % result)

519 520 521
    def tearDown(self):
        shutil.rmtree(self.tmpdir)

522

523 524 525
class RunnerHumanOutputTest(unittest.TestCase):

    def setUp(self):
526
        self.tmpdir = tempfile.mkdtemp(prefix='avocado_' + __name__)
527 528 529

    def test_output_pass(self):
        os.chdir(basedir)
530 531
        cmd_line = ('%s run --sysinfo=off --job-results-dir %s '
                    'passtest.py' % (AVOCADO, self.tmpdir))
532
        result = process.run(cmd_line, ignore_status=True)
533
        expected_rc = exit_codes.AVOCADO_ALL_OK
534 535 536 537 538 539 540
        self.assertEqual(result.exit_status, expected_rc,
                         "Avocado did not return rc %d:\n%s" %
                         (expected_rc, result))
        self.assertIn('passtest.py:PassTest.test:  PASS', result.stdout)

    def test_output_fail(self):
        os.chdir(basedir)
541 542
        cmd_line = ('%s run --sysinfo=off --job-results-dir %s '
                    'failtest.py' % (AVOCADO, self.tmpdir))
543
        result = process.run(cmd_line, ignore_status=True)
544
        expected_rc = exit_codes.AVOCADO_TESTS_FAIL
545 546 547 548 549 550 551
        self.assertEqual(result.exit_status, expected_rc,
                         "Avocado did not return rc %d:\n%s" %
                         (expected_rc, result))
        self.assertIn('failtest.py:FailTest.test:  FAIL', result.stdout)

    def test_output_error(self):
        os.chdir(basedir)
552 553
        cmd_line = ('%s run --sysinfo=off --job-results-dir %s '
                    'errortest.py' % (AVOCADO, self.tmpdir))
554
        result = process.run(cmd_line, ignore_status=True)
555
        expected_rc = exit_codes.AVOCADO_TESTS_FAIL
556 557 558 559 560
        self.assertEqual(result.exit_status, expected_rc,
                         "Avocado did not return rc %d:\n%s" %
                         (expected_rc, result))
        self.assertIn('errortest.py:ErrorTest.test:  ERROR', result.stdout)

A
Amador Pahim 已提交
561
    def test_output_cancel(self):
562
        os.chdir(basedir)
563 564
        cmd_line = ('%s run --sysinfo=off --job-results-dir %s '
                    'cancelonsetup.py' % (AVOCADO, self.tmpdir))
565
        result = process.run(cmd_line, ignore_status=True)
566
        expected_rc = exit_codes.AVOCADO_ALL_OK
567 568 569
        self.assertEqual(result.exit_status, expected_rc,
                         "Avocado did not return rc %d:\n%s" %
                         (expected_rc, result))
A
Amador Pahim 已提交
570 571
        self.assertIn('PASS 0 | ERROR 0 | FAIL 0 | SKIP 0 | WARN 0 | INTERRUPT 0 | CANCEL 1',
                      result.stdout)
572

573 574
    @unittest.skipIf(not GNU_ECHO_BINARY,
                     'GNU style echo binary not available')
575 576
    def test_ugly_echo_cmd(self):
        os.chdir(basedir)
577
        cmd_line = ('%s run --external-runner "%s -ne" '
578
                    '"foo\\\\\\n\\\'\\\\\\"\\\\\\nbar/baz" --job-results-dir %s'
A
Amador Pahim 已提交
579
                    ' --sysinfo=off  --show-job-log' %
580
                    (AVOCADO, GNU_ECHO_BINARY, self.tmpdir))
581 582 583 584 585
        result = process.run(cmd_line, ignore_status=True)
        expected_rc = exit_codes.AVOCADO_ALL_OK
        self.assertEqual(result.exit_status, expected_rc,
                         "Avocado did not return rc %s:\n%s" %
                         (expected_rc, result))
586 587 588
        self.assertIn('[stdout] foo', result.stdout, result)
        self.assertIn('[stdout] \'"', result.stdout, result)
        self.assertIn('[stdout] bar/baz', result.stdout, result)
589 590
        self.assertIn('PASS 1-foo\\\\n\\\'\\"\\\\nbar/baz',
                      result.stdout, result)
591 592 593 594 595 596 597
        # logdir name should escape special chars (/)
        test_dirs = glob.glob(os.path.join(self.tmpdir, 'latest',
                                           'test-results', '*'))
        self.assertEqual(len(test_dirs), 1, "There are multiple directories in"
                         " test-results dir, but only one test was executed: "
                         "%s" % (test_dirs))
        self.assertEqual(os.path.basename(test_dirs[0]),
598
                         '1-foo\\\\n\\\'\\"\\\\nbar_baz')
599

600
    def test_replay_skip_skipped(self):
601 602
        cmd = ("%s run --job-results-dir %s --json - "
               "cancelonsetup.py" % (AVOCADO, self.tmpdir))
603
        result = process.run(cmd)
604
        result = json.loads(result.stdout)
605
        jobid = str(result["job_id"])
606 607
        cmd = ("%s run --job-results-dir %s --replay %s "
               "--replay-test-status PASS" % (AVOCADO, self.tmpdir, jobid))
608
        process.run(cmd)
609

610 611 612
    def tearDown(self):
        shutil.rmtree(self.tmpdir)

613

614
class RunnerSimpleTest(unittest.TestCase):
615 616

    def setUp(self):
617
        self.tmpdir = tempfile.mkdtemp(prefix='avocado_' + __name__)
618
        self.pass_script = script.TemporaryScript(
619
            'ʊʋʉʈɑ ʅʛʌ',
620
            PASS_SCRIPT_CONTENTS,
621
            'avocado_simpletest_functional')
622
        self.pass_script.save()
L
Lukáš Doktor 已提交
623 624 625 626
        self.fail_script = script.TemporaryScript('avocado_fail.sh',
                                                  FAIL_SCRIPT_CONTENTS,
                                                  'avocado_simpletest_'
                                                  'functional')
627
        self.fail_script.save()
628

629
    def test_simpletest_pass(self):
630
        os.chdir(basedir)
631 632
        cmd_line = ('%s run --job-results-dir %s --sysinfo=off'
                    ' "%s"' % (AVOCADO, self.tmpdir, self.pass_script.path))
633
        result = process.run(cmd_line, ignore_status=True)
634
        expected_rc = exit_codes.AVOCADO_ALL_OK
635 636 637 638
        self.assertEqual(result.exit_status, expected_rc,
                         "Avocado did not return rc %d:\n%s" %
                         (expected_rc, result))

639
    def test_simpletest_fail(self):
640
        os.chdir(basedir)
641 642
        cmd_line = ('%s run --job-results-dir %s --sysinfo=off'
                    ' %s' % (AVOCADO, self.tmpdir, self.fail_script.path))
643
        result = process.run(cmd_line, ignore_status=True)
644
        expected_rc = exit_codes.AVOCADO_TESTS_FAIL
645 646 647 648
        self.assertEqual(result.exit_status, expected_rc,
                         "Avocado did not return rc %d:\n%s" %
                         (expected_rc, result))

649 650
    def test_runner_onehundred_fail_timing(self):
        """
A
Amador Pahim 已提交
651
        We can be pretty sure that a failtest should return immediately. Let's
652
        run 100 of them and assure they not take more than 30 seconds to run.
653

654 655
        Notice: on a current machine this takes about 0.12s, so 30 seconds is
        considered to be pretty safe here.
656 657
        """
        os.chdir(basedir)
658
        one_hundred = 'failtest.py ' * 100
659 660
        cmd_line = ('%s run --job-results-dir %s --sysinfo=off %s'
                    % (AVOCADO, self.tmpdir, one_hundred))
661 662 663
        initial_time = time.time()
        result = process.run(cmd_line, ignore_status=True)
        actual_time = time.time() - initial_time
664
        self.assertLess(actual_time, 30.0)
665
        expected_rc = exit_codes.AVOCADO_TESTS_FAIL
666 667 668 669 670 671 672 673 674
        self.assertEqual(result.exit_status, expected_rc,
                         "Avocado did not return rc %d:\n%s" % (expected_rc, result))

    def test_runner_sleep_fail_sleep_timing(self):
        """
        Sleeptest is supposed to take 1 second, let's make a sandwich of
        100 failtests and check the test runner timing.
        """
        os.chdir(basedir)
675 676
        sleep_fail_sleep = ('sleeptest.py ' + 'failtest.py ' * 100 +
                            'sleeptest.py')
677 678
        cmd_line = ('%s run --job-results-dir %s --sysinfo=off %s'
                    % (AVOCADO, self.tmpdir, sleep_fail_sleep))
679 680 681
        initial_time = time.time()
        result = process.run(cmd_line, ignore_status=True)
        actual_time = time.time() - initial_time
682
        self.assertLess(actual_time, 33.0)
683
        expected_rc = exit_codes.AVOCADO_TESTS_FAIL
684 685 686
        self.assertEqual(result.exit_status, expected_rc,
                         "Avocado did not return rc %d:\n%s" % (expected_rc, result))

687 688 689 690 691
    def test_simplewarning(self):
        """
        simplewarning.sh uses the avocado-bash-utils
        """
        os.chdir(basedir)
692 693 694
        cmd_line = ('%s run --job-results-dir %s --sysinfo=off '
                    'examples/tests/simplewarning.sh --show-job-log'
                    % (AVOCADO, self.tmpdir))
695
        result = process.run(cmd_line, ignore_status=True)
696 697 698 699
        expected_rc = exit_codes.AVOCADO_ALL_OK
        self.assertEqual(result.exit_status, expected_rc,
                         "Avocado did not return rc %s:\n%s" %
                         (expected_rc, result))
700 701
        self.assertIn('DEBUG| Debug message', result.stdout, result)
        self.assertIn('INFO | Info message', result.stdout, result)
702
        self.assertIn('WARN | Warning message (should cause this test to '
703
                      'finish with warning)', result.stdout, result)
704
        self.assertIn('ERROR| Error message (ordinary message not changing '
705
                      'the results)', result.stdout, result)
706

707 708 709 710 711 712
    def test_non_absolute_path(self):
        avocado_path = os.path.join(basedir, 'scripts', 'avocado')
        test_base_dir = os.path.dirname(self.pass_script.path)
        test_file_name = os.path.basename(self.pass_script.path)
        os.chdir(test_base_dir)
        cmd_line = ('%s run --job-results-dir %s --sysinfo=off'
713
                    ' "%s"' % (avocado_path, self.tmpdir, test_file_name))
714 715 716 717 718 719
        result = process.run(cmd_line, ignore_status=True)
        expected_rc = exit_codes.AVOCADO_ALL_OK
        self.assertEqual(result.exit_status, expected_rc,
                         "Avocado did not return rc %d:\n%s" %
                         (expected_rc, result))

A
Amador Pahim 已提交
720
    @unittest.skipIf(not SLEEP_BINARY, 'sleep binary not available')
721
    def test_kill_stopped_sleep(self):
722 723 724 725
        proc = aexpect.Expect("%s run 60 --job-results-dir %s "
                              "--external-runner %s --sysinfo=off "
                              "--job-timeout 3"
                              % (AVOCADO, self.tmpdir, SLEEP_BINARY))
726 727
        proc.read_until_output_matches(["\(1/1\)"], timeout=3,
                                       internal_timeout=0.01)
728 729 730 731
        # We need pid of the avocado process, not the shell executing it
        avocado_shell = psutil.Process(proc.get_pid())
        avocado_proc = avocado_shell.children()[0]
        pid = avocado_proc.pid
732
        os.kill(pid, signal.SIGTSTP)   # This freezes the process
733
        deadline = time.time() + 9
734 735 736
        while time.time() < deadline:
            if not proc.is_alive():
                break
737
            time.sleep(0.1)
738 739
        else:
            proc.kill(signal.SIGKILL)
740
            self.fail("Avocado process still alive 5s after job-timeout:\n%s"
741 742 743 744 745 746 747
                      % proc.get_output())
        output = proc.get_output()
        self.assertIn("ctrl+z pressed, stopping test", output, "SIGTSTP "
                      "message not in the output, test was probably not "
                      "stopped.")
        self.assertIn("TIME", output, "TIME not in the output, avocado "
                      "probably died unexpectadly")
748
        self.assertEqual(proc.get_status(), 8, "Avocado did not finish with "
749
                         "1.")
750 751

        sleep_dir = astring.string_to_safe_path("1-60")
752
        debug_log = os.path.join(self.tmpdir, "latest", "test-results",
753
                                 sleep_dir, "debug.log")
754
        debug_log = open(debug_log).read()
755 756 757 758 759 760 761
        self.assertIn("Runner error occurred: Timeout reached", debug_log,
                      "Runner error occurred: Timeout reached message not "
                      "in the test's debug.log:\n%s" % debug_log)
        self.assertNotIn("Traceback (most recent", debug_log, "Traceback "
                         "present in the test's debug.log file, but it was "
                         "suppose to be stopped and unable to produce it.\n"
                         "%s" % debug_log)
762

763
    def tearDown(self):
764 765
        self.pass_script.remove()
        self.fail_script.remove()
766
        shutil.rmtree(self.tmpdir)
767 768


769
class ExternalRunnerTest(unittest.TestCase):
C
Cleber Rosa 已提交
770 771

    def setUp(self):
772
        self.tmpdir = tempfile.mkdtemp(prefix='avocado_' + __name__)
C
Cleber Rosa 已提交
773 774 775
        self.pass_script = script.TemporaryScript(
            'pass',
            PASS_SHELL_CONTENTS,
776
            'avocado_externalrunner_functional')
C
Cleber Rosa 已提交
777 778 779 780
        self.pass_script.save()
        self.fail_script = script.TemporaryScript(
            'fail',
            FAIL_SHELL_CONTENTS,
781
            'avocado_externalrunner_functional')
C
Cleber Rosa 已提交
782 783
        self.fail_script.save()

784
    def test_externalrunner_pass(self):
C
Cleber Rosa 已提交
785
        os.chdir(basedir)
786 787 788
        cmd_line = ('%s run --job-results-dir %s --sysinfo=off '
                    '--external-runner=/bin/sh %s'
                    % (AVOCADO, self.tmpdir, self.pass_script.path))
C
Cleber Rosa 已提交
789
        result = process.run(cmd_line, ignore_status=True)
790
        expected_rc = exit_codes.AVOCADO_ALL_OK
C
Cleber Rosa 已提交
791 792 793 794
        self.assertEqual(result.exit_status, expected_rc,
                         "Avocado did not return rc %d:\n%s" %
                         (expected_rc, result))

795
    def test_externalrunner_fail(self):
C
Cleber Rosa 已提交
796
        os.chdir(basedir)
797 798 799
        cmd_line = ('%s run --job-results-dir %s --sysinfo=off '
                    '--external-runner=/bin/sh %s'
                    % (AVOCADO, self.tmpdir, self.fail_script.path))
C
Cleber Rosa 已提交
800
        result = process.run(cmd_line, ignore_status=True)
801
        expected_rc = exit_codes.AVOCADO_TESTS_FAIL
C
Cleber Rosa 已提交
802 803 804 805
        self.assertEqual(result.exit_status, expected_rc,
                         "Avocado did not return rc %d:\n%s" %
                         (expected_rc, result))

806
    def test_externalrunner_chdir_no_testdir(self):
C
Cleber Rosa 已提交
807
        os.chdir(basedir)
808 809 810
        cmd_line = ('%s run --job-results-dir %s --sysinfo=off '
                    '--external-runner=/bin/sh --external-runner-chdir=test %s'
                    % (AVOCADO, self.tmpdir, self.pass_script.path))
C
Cleber Rosa 已提交
811
        result = process.run(cmd_line, ignore_status=True)
812 813
        expected_output = ('Option "--external-runner-chdir=test" requires '
                           '"--external-runner-testdir" to be set')
C
Cleber Rosa 已提交
814
        self.assertIn(expected_output, result.stderr)
815
        expected_rc = exit_codes.AVOCADO_JOB_FAIL
816 817 818 819 820 821
        self.assertEqual(result.exit_status, expected_rc,
                         "Avocado did not return rc %d:\n%s" %
                         (expected_rc, result))

    def test_externalrunner_no_url(self):
        os.chdir(basedir)
822 823
        cmd_line = ('%s run --job-results-dir %s --sysinfo=off '
                    '--external-runner=%s' % (AVOCADO, self.tmpdir, TRUE_CMD))
824
        result = process.run(cmd_line, ignore_status=True)
825 826
        expected_output = ('No test references provided nor any other '
                           'arguments resolved into tests')
827 828
        self.assertIn(expected_output, result.stderr)
        expected_rc = exit_codes.AVOCADO_JOB_FAIL
C
Cleber Rosa 已提交
829 830 831 832 833 834 835 836 837 838
        self.assertEqual(result.exit_status, expected_rc,
                         "Avocado did not return rc %d:\n%s" %
                         (expected_rc, result))

    def tearDown(self):
        self.pass_script.remove()
        self.fail_script.remove()
        shutil.rmtree(self.tmpdir)


839
class AbsPluginsTest(object):
840

841
    def setUp(self):
842
        self.base_outputdir = tempfile.mkdtemp(prefix='avocado_' + __name__)
843

844 845 846 847 848 849
    def tearDown(self):
        shutil.rmtree(self.base_outputdir)


class PluginsTest(AbsPluginsTest, unittest.TestCase):

850 851
    def test_sysinfo_plugin(self):
        os.chdir(basedir)
852
        cmd_line = '%s sysinfo %s' % (AVOCADO, self.base_outputdir)
853
        result = process.run(cmd_line, ignore_status=True)
854
        expected_rc = exit_codes.AVOCADO_ALL_OK
855 856 857 858 859 860
        self.assertEqual(result.exit_status, expected_rc,
                         "Avocado did not return rc %d:\n%s" %
                         (expected_rc, result))
        sysinfo_files = os.listdir(self.base_outputdir)
        self.assertGreater(len(sysinfo_files), 0, "Empty sysinfo files dir")

861 862
    def test_list_plugin(self):
        os.chdir(basedir)
863
        cmd_line = '%s list' % AVOCADO
864 865
        result = process.run(cmd_line, ignore_status=True)
        output = result.stdout
866
        expected_rc = exit_codes.AVOCADO_ALL_OK
867 868 869 870 871
        self.assertEqual(result.exit_status, expected_rc,
                         "Avocado did not return rc %d:\n%s" %
                         (expected_rc, result))
        self.assertNotIn('No tests were found on current tests dir', output)

872 873
    def test_list_error_output(self):
        os.chdir(basedir)
874
        cmd_line = '%s list sbrubles' % AVOCADO
875 876
        result = process.run(cmd_line, ignore_status=True)
        output = result.stderr
877
        expected_rc = exit_codes.AVOCADO_FAIL
878 879 880
        self.assertEqual(result.exit_status, expected_rc,
                         "Avocado did not return rc %d:\n%s" %
                         (expected_rc, result))
881
        self.assertIn("Unable to resolve reference", output)
882

883 884 885 886 887 888 889 890 891 892 893 894 895
    def test_list_no_file_loader(self):
        os.chdir(basedir)
        cmd_line = ("%s list --loaders external --verbose -- "
                    "this-wont-be-matched" % AVOCADO)
        result = process.run(cmd_line, ignore_status=True)
        self.assertEqual(result.exit_status, exit_codes.AVOCADO_ALL_OK,
                         "Avocado did not return rc %d:\n%s"
                         % (exit_codes.AVOCADO_ALL_OK, result))
        exp = ("Type    Test\nMISSING this-wont-be-matched\n\nEXTERNAL: 0\n"
               "MISSING: 1\n")
        self.assertEqual(exp, result.stdout, "Stdout mismatch:\n%s\n\n%s"
                         % (exp, result))

896 897
    def test_plugin_list(self):
        os.chdir(basedir)
898
        cmd_line = '%s plugins' % AVOCADO
899 900
        result = process.run(cmd_line, ignore_status=True)
        output = result.stdout
901
        expected_rc = exit_codes.AVOCADO_ALL_OK
902 903 904
        self.assertEqual(result.exit_status, expected_rc,
                         "Avocado did not return rc %d:\n%s" %
                         (expected_rc, result))
905 906
        if sys.version_info[:2] >= (2, 7, 0):
            self.assertNotIn('Disabled', output)
907

908
    def test_config_plugin(self):
909
        os.chdir(basedir)
910
        cmd_line = '%s config --paginator off' % AVOCADO
911 912
        result = process.run(cmd_line, ignore_status=True)
        output = result.stdout
913
        expected_rc = exit_codes.AVOCADO_ALL_OK
914 915 916 917 918 919 920
        self.assertEqual(result.exit_status, expected_rc,
                         "Avocado did not return rc %d:\n%s" %
                         (expected_rc, result))
        self.assertNotIn('Disabled', output)

    def test_config_plugin_datadir(self):
        os.chdir(basedir)
921
        cmd_line = '%s config --datadir --paginator off' % AVOCADO
922 923
        result = process.run(cmd_line, ignore_status=True)
        output = result.stdout
924
        expected_rc = exit_codes.AVOCADO_ALL_OK
925 926 927 928 929
        self.assertEqual(result.exit_status, expected_rc,
                         "Avocado did not return rc %d:\n%s" %
                         (expected_rc, result))
        self.assertNotIn('Disabled', output)

930 931
    def test_disable_plugin(self):
        os.chdir(basedir)
932
        cmd_line = '%s plugins' % AVOCADO
933 934 935 936 937 938 939 940 941 942 943
        result = process.run(cmd_line, ignore_status=True)
        expected_rc = exit_codes.AVOCADO_ALL_OK
        self.assertEqual(result.exit_status, expected_rc,
                         "Avocado did not return rc %d:\n%s" %
                         (expected_rc, result))
        self.assertIn("Collect system information", result.stdout)

        config_content = "[plugins]\ndisable=['cli.cmd.sysinfo',]"
        config = script.TemporaryScript("disable_sysinfo_cmd.conf",
                                        config_content)
        with config:
944
            cmd_line = '%s --config %s plugins' % (AVOCADO, config)
945 946 947 948 949 950 951
            result = process.run(cmd_line, ignore_status=True)
            expected_rc = exit_codes.AVOCADO_ALL_OK
            self.assertEqual(result.exit_status, expected_rc,
                             "Avocado did not return rc %d:\n%s" %
                             (expected_rc, result))
            self.assertNotIn("Collect system information", result.stdout)

952 953 954 955 956 957 958 959 960 961 962 963
    def test_plugin_order(self):
        """
        Tests plugin order by configuration file

        First it checks if html, json, xunit and zip_archive plugins are enabled.
        Then it runs a test with zip_archive running first, which means the html,
        json and xunit output files do not make into the archive.

        Then it runs with zip_archive set to run last, which means the html,
        json and xunit output files *do* make into the archive.
        """
        def run_config(config_path):
964
            cmd = ('%s --config %s run passtest.py --archive '
965
                   '--job-results-dir %s --sysinfo=off'
966
                   % (AVOCADO, config_path, self.base_outputdir))
967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982
            result = process.run(cmd, ignore_status=True)
            expected_rc = exit_codes.AVOCADO_ALL_OK
            self.assertEqual(result.exit_status, expected_rc,
                             "Avocado did not return rc %d:\n%s" %
                             (expected_rc, result))

        result_plugins = ["json", "xunit", "zip_archive"]
        result_outputs = ["results.json", "results.xml"]
        try:
            pkg_resources.require('avocado_result_html')
            result_plugins.append("html")
            result_outputs.append("html/results.html")
        except pkg_resources.DistributionNotFound:
            pass

        os.chdir(basedir)
983
        cmd_line = '%s plugins' % AVOCADO
984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005
        result = process.run(cmd_line, ignore_status=True)
        expected_rc = exit_codes.AVOCADO_ALL_OK
        self.assertEqual(result.exit_status, expected_rc,
                         "Avocado did not return rc %d:\n%s" %
                         (expected_rc, result))
        for result_plugin in result_plugins:
            self.assertIn(result_plugin, result.stdout)

        config_content_zip_first = "[plugins.result]\norder=['zip_archive']"
        config_zip_first = script.TemporaryScript("zip_first.conf",
                                                  config_content_zip_first)
        with config_zip_first:
            run_config(config_zip_first)
            archives = glob.glob(os.path.join(self.base_outputdir, '*.zip'))
            self.assertEqual(len(archives), 1, "ZIP Archive not generated")
            zip_file = zipfile.ZipFile(archives[0], 'r')
            zip_file_list = zip_file.namelist()
            for result_output in result_outputs:
                self.assertNotIn(result_output, zip_file_list)
            os.unlink(archives[0])

        config_content_zip_last = ("[plugins.result]\norder=['html', 'json',"
1006 1007
                                   "'xunit', 'non_existing_plugin_is_ignored'"
                                   ",'zip_archive']")
1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018
        config_zip_last = script.TemporaryScript("zip_last.conf",
                                                 config_content_zip_last)
        with config_zip_last:
            run_config(config_zip_last)
            archives = glob.glob(os.path.join(self.base_outputdir, '*.zip'))
            self.assertEqual(len(archives), 1, "ZIP Archive not generated")
            zip_file = zipfile.ZipFile(archives[0], 'r')
            zip_file_list = zip_file.namelist()
            for result_output in result_outputs:
                self.assertIn(result_output, zip_file_list)

1019 1020
    def test_Namespace_object_has_no_attribute(self):
        os.chdir(basedir)
1021
        cmd_line = '%s plugins' % AVOCADO
1022 1023
        result = process.run(cmd_line, ignore_status=True)
        output = result.stderr
1024
        expected_rc = exit_codes.AVOCADO_ALL_OK
1025 1026 1027 1028 1029
        self.assertEqual(result.exit_status, expected_rc,
                         "Avocado did not return rc %d:\n%s" %
                         (expected_rc, result))
        self.assertNotIn("'Namespace' object has no attribute", output)

1030

1031 1032 1033 1034
class ParseXMLError(Exception):
    pass


1035
class PluginsXunitTest(AbsPluginsTest, unittest.TestCase):
1036

1037
    def setUp(self):
1038
        self.tmpdir = tempfile.mkdtemp(prefix='avocado_' + __name__)
1039 1040
        self.junit = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                     os.path.pardir, ".data", 'junit-4.xsd'))
1041 1042
        super(PluginsXunitTest, self).setUp()

1043
    def run_and_check(self, testname, e_rc, e_ntests, e_nerrors,
1044
                      e_nnotfound, e_nfailures, e_nskip):
1045
        os.chdir(basedir)
1046 1047
        cmd_line = ('%s run --job-results-dir %s --sysinfo=off'
                    ' --xunit - %s' % (AVOCADO, self.tmpdir, testname))
1048 1049 1050 1051 1052 1053 1054
        result = process.run(cmd_line, ignore_status=True)
        xml_output = result.stdout
        self.assertEqual(result.exit_status, e_rc,
                         "Avocado did not return rc %d:\n%s" %
                         (e_rc, result))
        try:
            xunit_doc = xml.dom.minidom.parseString(xml_output)
1055
        except Exception as detail:
1056 1057 1058
            raise ParseXMLError("Failed to parse content: %s\n%s" %
                                (detail, xml_output))

1059 1060 1061 1062 1063 1064 1065 1066
        with open(self.junit, 'r') as f:
            xmlschema = etree.XMLSchema(etree.parse(f))

        self.assertTrue(xmlschema.validate(etree.parse(StringIO(xml_output))),
                        "Failed to validate against %s, message:\n%s" %
                        (self.junit,
                         xmlschema.error_log.filter_from_errors()))

1067 1068 1069 1070
        testsuite_list = xunit_doc.getElementsByTagName('testsuite')
        self.assertEqual(len(testsuite_list), 1, 'More than one testsuite tag')

        testsuite_tag = testsuite_list[0]
1071 1072
        self.assertEqual(len(testsuite_tag.attributes), 7,
                         'The testsuite tag does not have 7 attributes. '
1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089
                         'XML:\n%s' % xml_output)

        n_tests = int(testsuite_tag.attributes['tests'].value)
        self.assertEqual(n_tests, e_ntests,
                         "Unexpected number of executed tests, "
                         "XML:\n%s" % xml_output)

        n_errors = int(testsuite_tag.attributes['errors'].value)
        self.assertEqual(n_errors, e_nerrors,
                         "Unexpected number of test errors, "
                         "XML:\n%s" % xml_output)

        n_failures = int(testsuite_tag.attributes['failures'].value)
        self.assertEqual(n_failures, e_nfailures,
                         "Unexpected number of test failures, "
                         "XML:\n%s" % xml_output)

1090
        n_skip = int(testsuite_tag.attributes['skipped'].value)
1091 1092 1093 1094
        self.assertEqual(n_skip, e_nskip,
                         "Unexpected number of test skips, "
                         "XML:\n%s" % xml_output)

1095
    def test_xunit_plugin_passtest(self):
1096
        self.run_and_check('passtest.py', exit_codes.AVOCADO_ALL_OK,
1097
                           1, 0, 0, 0, 0)
1098 1099

    def test_xunit_plugin_failtest(self):
1100
        self.run_and_check('failtest.py', exit_codes.AVOCADO_TESTS_FAIL,
1101
                           1, 0, 0, 1, 0)
1102

1103
    def test_xunit_plugin_skiponsetuptest(self):
A
Amador Pahim 已提交
1104
        self.run_and_check('cancelonsetup.py', exit_codes.AVOCADO_ALL_OK,
1105
                           1, 0, 0, 0, 1)
1106

1107
    def test_xunit_plugin_errortest(self):
1108
        self.run_and_check('errortest.py', exit_codes.AVOCADO_TESTS_FAIL,
1109
                           1, 1, 0, 0, 0)
1110

1111 1112 1113 1114
    def tearDown(self):
        shutil.rmtree(self.tmpdir)
        super(PluginsXunitTest, self).tearDown()

1115 1116 1117 1118 1119

class ParseJSONError(Exception):
    pass


1120
class PluginsJSONTest(AbsPluginsTest, unittest.TestCase):
1121

1122
    def setUp(self):
1123
        self.tmpdir = tempfile.mkdtemp(prefix='avocado_' + __name__)
1124 1125
        super(PluginsJSONTest, self).setUp()

1126
    def run_and_check(self, testname, e_rc, e_ntests, e_nerrors,
1127
                      e_nfailures, e_nskip, e_ncancel=0, external_runner=None):
1128
        os.chdir(basedir)
1129 1130
        cmd_line = ('%s run --job-results-dir %s --sysinfo=off --json - '
                    '--archive %s' % (AVOCADO, self.tmpdir, testname))
1131 1132
        if external_runner is not None:
            cmd_line += " --external-runner '%s'" % external_runner
1133 1134 1135 1136 1137 1138 1139
        result = process.run(cmd_line, ignore_status=True)
        json_output = result.stdout
        self.assertEqual(result.exit_status, e_rc,
                         "Avocado did not return rc %d:\n%s" %
                         (e_rc, result))
        try:
            json_data = json.loads(json_output)
1140
        except Exception as detail:
1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157
            raise ParseJSONError("Failed to parse content: %s\n%s" %
                                 (detail, json_output))
        self.assertTrue(json_data, "Empty JSON result:\n%s" % json_output)
        self.assertIsInstance(json_data['tests'], list,
                              "JSON result lacks 'tests' list")
        n_tests = len(json_data['tests'])
        self.assertEqual(n_tests, e_ntests,
                         "Different number of expected tests")
        n_errors = json_data['errors']
        self.assertEqual(n_errors, e_nerrors,
                         "Different number of expected tests")
        n_failures = json_data['failures']
        self.assertEqual(n_failures, e_nfailures,
                         "Different number of expected tests")
        n_skip = json_data['skip']
        self.assertEqual(n_skip, e_nskip,
                         "Different number of skipped tests")
1158 1159
        n_cancel = json_data['cancel']
        self.assertEqual(n_cancel, e_ncancel)
1160
        return json_data
1161

1162
    def test_json_plugin_passtest(self):
1163
        self.run_and_check('passtest.py', exit_codes.AVOCADO_ALL_OK,
1164
                           1, 0, 0, 0)
1165 1166

    def test_json_plugin_failtest(self):
1167
        self.run_and_check('failtest.py', exit_codes.AVOCADO_TESTS_FAIL,
1168
                           1, 0, 1, 0)
1169

1170
    def test_json_plugin_skiponsetuptest(self):
A
Amador Pahim 已提交
1171
        self.run_and_check('cancelonsetup.py', exit_codes.AVOCADO_ALL_OK,
1172
                           1, 0, 0, 0, 1)
1173

1174
    def test_json_plugin_errortest(self):
1175
        self.run_and_check('errortest.py', exit_codes.AVOCADO_TESTS_FAIL,
1176
                           1, 1, 0, 0)
1177

1178
    @unittest.skipIf(not GNU_ECHO_BINARY, 'echo binary not available')
1179
    def test_ugly_echo_cmd(self):
1180
        data = self.run_and_check('"-ne foo\\\\\\n\\\'\\\\\\"\\\\\\'
1181
                                  'nbar/baz"', exit_codes.AVOCADO_ALL_OK, 1, 0,
1182
                                  0, 0, external_runner=GNU_ECHO_BINARY)
1183 1184
        # The executed test should be this
        self.assertEqual(data['tests'][0]['url'],
1185
                         '1--ne foo\\\\n\\\'\\"\\\\nbar/baz')
1186 1187
        # logdir name should escape special chars (/)
        self.assertEqual(os.path.basename(data['tests'][0]['logdir']),
1188
                         '1--ne foo\\\\n\\\'\\"\\\\nbar_baz')
1189

1190 1191 1192 1193
    def tearDown(self):
        shutil.rmtree(self.tmpdir)
        super(PluginsJSONTest, self).tearDown()

L
Lukáš Doktor 已提交
1194

1195 1196
if __name__ == '__main__':
    unittest.main()