Reorganize unittests and functional tests

In order for people to run all avocado tests at once,
move and reorganize tests under a directory called
selftests. For someone to run all self tests in
avocado, one would have to execute:

    selftests/run selftests/all

This is a shortcut for developers. .travis.yml
was also updated to run tests based on the new
directory structure.
Signed-off-by: NLucas Meneghel Rodrigues <lmr@redhat.com>

.travis.yml

@@ -12,21 +12,8 @@ install:
     - pip install -r requirements.txt
 
 script:
-    - ./unittests/runtests.py -c .nose.cfg
     - inspekt lint
     - inspekt style
-    - make -C docs html 2>&1 | grep -E '(ERROR|WARNING)' || test $? -eq 1
-    - ./scripts/avocado run "sleeptest sleeptest"
-    - ./scripts/avocado run "sleeptest failtest sleeptest" || test $? -eq 1
-    - ./scripts/avocado run "bogustest" || test $? -ne 3
-    - export PYTHONPATH=$PYTHONPATH:.
-    - ./tests/sleeptest/sleeptest.py
-    - ./tests/skiptest/skiptest.py
-    - ./tests/failtest/failtest.py || test $? -eq 1
-    - ./tests/errortest/errortest.py || test $? -eq 1
-    - ./tests/warntest/warntest.py || test $? -eq 1
-    - ./scripts/avocado run --multiplex tests/sleeptest/sleeptest.mplx
-    - ./scripts/avocado run "sleeptest sleeptest" --multiplex tests/sleeptest/sleeptest.mplx
-    - ./scripts/avocado run "sleeptest failtest" --multiplex tests/sleeptest/sleeptest.mplx || test $? -eq 1
-    - ./scripts/avocado multiplex tests/sleeptest/sleeptest.mplx
-    - ./scripts/avocado multiplex nonexist || test $? -eq 2
+    - ./selftests/run -v selftests/all/doc
+    - ./selftests/run -v selftests/all/functional
+    - ./selftests/run selftests/all/unit -c selftests/.nose.cfg

selftests/all/doc/doc_build_test.py

+#!/usr/bin/python
+"""
+Build documentation and report whether we had warning/error messages.
+
+This is geared towards documentation build regression testing.
+"""
+import os
+import sys
+
+# simple magic for using scripts within a source tree
+basedir = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', '..')
+basedir = os.path.abspath(basedir)
+if os.path.isdir(os.path.join(basedir, 'avocado')):
+    sys.path.append(basedir)
+
+from avocado.utils import process
+
+
+class DocBuildError(Exception):
+    pass
+
+
+def test_build_docs():
+    """
+    Build avocado HTML docs, reporting failures
+    """
+    ignore_list = []
+    failure_lines = []
+    doc_dir = os.path.join(basedir, 'docs')
+    process.run('make -C %s clean' % doc_dir)
+    result = process.run('make -C %s html' % doc_dir)
+    stdout = result.stdout.splitlines()
+    stderr = result.stderr.splitlines()
+    output_lines = stdout + stderr
+    for line in output_lines:
+        ignore_msg = False
+        for ignore in ignore_list:
+            if ignore in line:
+                print 'Expected warning ignored: %s' % line
+                ignore_msg = True
+        if ignore_msg:
+            continue
+        if 'ERROR' in line:
+            failure_lines.append(line)
+        if 'WARNING' in line:
+            failure_lines.append(line)
+    if failure_lines:
+        e_msg = ('%s ERRORS and/or WARNINGS detected while building the html docs:\n' %
+                 len(failure_lines))
+        for (index, failure_line) in enumerate(failure_lines):
+            e_msg += "%s) %s\n" % (index + 1, failure_line)
+        e_msg += 'Please check the output and fix your docstrings/.rst docs'
+        raise DocBuildError(e_msg)
+
+if __name__ == '__main__':
+    test_build_docs()

selftests/all/functional/avocado/basic_tests.py

+#!/usr/bin/env python
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See LICENSE for more details.
+#
+# Copyright: Red Hat Inc. 2013-2014
+# Author: Lucas Meneghel Rodrigues <lmr@redhat.com>
+
+import unittest
+import os
+import sys
+
+# simple magic for using scripts within a source tree
+basedir = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', '..', '..')
+basedir = os.path.abspath(basedir)
+if os.path.isdir(os.path.join(basedir, 'avocado')):
+    sys.path.append(basedir)
+
+from avocado.utils import process
+
+
+class RunnerOperationTest(unittest.TestCase):
+
+    def test_runner_all_ok(self):
+        os.chdir(basedir)
+        cmd_line = './scripts/avocado run "sleeptest sleeptest"'
+        process.run(cmd_line)
+
+    def test_runner_tests_fail(self):
+        os.chdir(basedir)
+        cmd_line = './scripts/avocado run "sleeptest failtest sleeptest"'
+        result = process.run(cmd_line, ignore_status=True)
+        expected_rc = 1
+        self.assertEqual(result.exit_status, expected_rc,
+                         "Avocado did not return rc %d:\n%s" % (expected_rc, result))
+
+    def test_runner_nonexistent_test(self):
+        os.chdir(basedir)
+        cmd_line = './scripts/avocado run bogustest'
+        result = process.run(cmd_line, ignore_status=True)
+        expected_rc = 1
+        unexpected_rc = 3
+        self.assertNotEqual(result.exit_status, unexpected_rc,
+                            "Avocado crashed (rc %d):\n%s" % (unexpected_rc, result))
+        self.assertEqual(result.exit_status, expected_rc,
+                         "Avocado did not return rc %d:\n%s" % (expected_rc, result))
+
+if __name__ == '__main__':
+    unittest.main()

selftests/all/functional/avocado/multiplex_tests.py

+#!/usr/bin/env python
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See LICENSE for more details.
+#
+# Copyright: Red Hat Inc. 2013-2014
+# Author: Lucas Meneghel Rodrigues <lmr@redhat.com>
+
+import unittest
+import os
+import sys
+
+# simple magic for using scripts within a source tree
+basedir = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', '..', '..')
+basedir = os.path.abspath(basedir)
+if os.path.isdir(os.path.join(basedir, 'avocado')):
+    sys.path.append(basedir)
+
+from avocado.utils import process
+
+
+class MultiplexTests(unittest.TestCase):
+
+    def run_and_check(self, cmd_line, expected_rc):
+        os.chdir(basedir)
+        result = process.run(cmd_line, ignore_status=True)
+        self.assertEqual(result.exit_status, expected_rc,
+                         "Command %s did not return rc "
+                         "%d:\n%s" % (cmd_line, expected_rc, result))
+
+    def test_mplex_plugin(self):
+        cmd_line = './scripts/avocado multiplex tests/sleeptest/sleeptest.mplx'
+        expected_rc = 0
+        self.run_and_check(cmd_line, expected_rc)
+
+    def test_mplex_plugin_nonexistent(self):
+        cmd_line = './scripts/avocado multiplex nonexist'
+        expected_rc = 2
+        self.run_and_check(cmd_line, expected_rc)
+
+    def test_run_mplex_sleeptest(self):
+        cmd_line = './scripts/avocado run sleeptest --multiplex tests/sleeptest/sleeptest.mplx'
+        expected_rc = 0
+        self.run_and_check(cmd_line, expected_rc)
+
+    def test_run_mplex_doublesleep(self):
+        cmd_line = './scripts/avocado run "sleeptest sleeptest" --multiplex tests/sleeptest/sleeptest.mplx'
+        expected_rc = 0
+        self.run_and_check(cmd_line, expected_rc)
+
+    def test_run_mplex_failtest(self):
+        cmd_line = './scripts/avocado run "sleeptest failtest" --multiplex tests/sleeptest/sleeptest.mplx'
+        expected_rc = 1
+        self.run_and_check(cmd_line, expected_rc)
+
+if __name__ == '__main__':
+    unittest.main()

selftests/all/functional/avocado/standalone_tests.py

+#!/usr/bin/env python
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See LICENSE for more details.
+#
+# Copyright: Red Hat Inc. 2013-2014
+# Author: Lucas Meneghel Rodrigues <lmr@redhat.com>
+
+import unittest
+import os
+import sys
+
+# simple magic for using scripts within a source tree
+basedir = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', '..', '..')
+basedir = os.path.abspath(basedir)
+if os.path.isdir(os.path.join(basedir, 'avocado')):
+    sys.path.append(basedir)
+
+from avocado.utils import process
+
+
+class StandaloneTests(unittest.TestCase):
+
+    def setUp(self):
+        self.original_pypath = os.environ.get('PYTHONPATH')
+        if self.original_pypath is not None:
+            os.environ['PYTHONPATH'] = '%s:%s' % (basedir, self.original_pypath)
+        else:
+            os.environ['PYTHONPATH'] = '%s' % basedir
+
+    def run_and_check(self, cmd_line, expected_rc, tstname):
+        os.chdir(basedir)
+        result = process.run(cmd_line, ignore_status=True)
+        self.assertEqual(result.exit_status, expected_rc,
+                         "Stand alone %s did not return rc "
+                         "%d:\n%s" % (tstname, expected_rc, result))
+
+    def test_sleeptest(self):
+        cmd_line = './tests/sleeptest/sleeptest.py'
+        expected_rc = 0
+        self.run_and_check(cmd_line, expected_rc, 'sleeptest')
+
+    def test_skiptest(self):
+        cmd_line = './tests/skiptest/skiptest.py'
+        expected_rc = 0
+        self.run_and_check(cmd_line, expected_rc, 'skiptest')
+
+    def test_failtest(self):
+        cmd_line = './tests/failtest/failtest.py'
+        expected_rc = 1
+        self.run_and_check(cmd_line, expected_rc, 'failtest')
+
+    def test_errortest(self):
+        cmd_line = './tests/errortest/errortest.py'
+        expected_rc = 1
+        self.run_and_check(cmd_line, expected_rc, 'errortest')
+
+    def test_warntest(self):
+        cmd_line = './tests/warntest/warntest.py'
+        expected_rc = 1
+        self.run_and_check(cmd_line, expected_rc, 'warntest')
+
+if __name__ == '__main__':
+    unittest.main()

unittests/runtests.py → selftests/run

@@ -42,7 +42,7 @@ class AvocadoTestSelector(Selector):
         return True
 
     def wantFile(self, filename):
-        if not filename.endswith('_unittest.py'):
+        if not filename.endswith('.py'):
             return False
 
         skip_tests = []
@@ -82,17 +82,8 @@ class AvocadoTestRunner(Plugin):
     def prepareTestLoader(self, loader):
         loader.selector = AvocadoTestSelector(loader.config)
 
-
-def run_test():
+if __name__ == '__main__':
     nose.main(addplugins=[AvocadoTestRunner(),
                           AttributeSelector(),
                           Xunit(),
                           Coverage()])
-
-
-def main():
-    run_test()
-
-
-if __name__ == '__main__':
-    main()