[RFC] Introduce the autotest regression suite v3

Since autotest has been helping to test the Linux virt
technologies, why not let them help to test autotest?

This is an early preview of a test suite designed
to do functional regression testing of autotest
using vms, as well as execute both the unittests
and the pylint checkers on it, and very basic
initial verification of the autotest server features.

It wouldn't be possible without the many cleanups
in static checking run made during the past couple
of weeks, now we're to a point where autotest master
can give useful static checking reports.

Hopefully this code will be refactored, and coverage
expanded.

Changes from v2:
 * Far better error checking and progress
   verification.

Changes from v1:
 * Fix stupid mistake from one of the rebases I've
   made (client ip acquisition was made too early,
   resulting in a VMIPAddressMissingError right
   in the beginning of the test).
Signed-off-by: NLucas Meneghel Rodrigues <lmr@redhat.com>

tests/autotest_regression.py

+import logging
+from autotest.client.shared import error
+from virttest import aexpect, utils_misc
+
+
+@error.context_aware
+def run_autotest_regression(test, params, env):
+    """
+    Autotest regression test:
+
+    Use Virtual Machines to test autotest.
+
+    1) Clone the given guest OS (only Linux) image twice.
+    2) Boot 2 VMs (autotest_server_vm and autotest_client_vm)
+    4) Install the autotest server in the server vm
+    5) Run the unittests
+    6) Run the pylint checker
+    7) Run a simple client sleeptest
+    8) Run a simple server sleeptest
+    9) Register the client vm in the autotest server
+    10) Schedule a simple job sleeptest in the client. Wait for client reboot.
+    11) If any of these steps have failed, fail the test and report the error
+
+    @param test: virt test object
+    @param params: Dictionary with the test parameters
+    @param env: Dictionary with test environment.
+    """
+    step_failures = []
+    autotest_repo = params['autotest_repo']
+    autotest_branch = params['autotest_branch']
+    autotest_commit = params['autotest_commit']
+    password = params['password']
+    autotest_install_timeout = int(params.get('autotest_install_timeout', 1800))
+    unittests_run_timeout = int(params.get('unittests_run_timeout', 1800))
+    pylint_run_timeout = int(params.get('pylint_run_timeout', 1800))
+    vm_names = params["vms"].split()
+    server_name = vm_names[0]
+    client_name = vm_names[1]
+    vm_server = env.get_vm(server_name)
+    vm_server.verify_alive()
+    vm_client = env.get_vm(client_name)
+    vm_client.verify_alive()
+
+    timeout = float(params.get("login_timeout", 240))
+    session_server = vm_server.wait_for_login(timeout=timeout)
+    session_client = vm_client.wait_for_login(timeout=timeout)
+    client_ip = vm_client.get_address()
+    server_ip = vm_server.get_address()
+
+    step1 = "autotest-server-install"
+    try:
+        # Download the install script and execute it
+        download_cmd = ("wget https://raw.github.com/autotest/autotest/master"
+                        "/contrib/install-autotest-server.sh")
+        session_server.cmd(download_cmd)
+        permission_cmd = ("chmod +x install-autotest-server.sh")
+        session_server.cmd(permission_cmd)
+        install_cmd = ("./install-autotest-server.sh -u Aut0t3st -d Aut0t3st "
+                       "-g %s -b %s" % (autotest_repo, autotest_branch))
+        if autotest_commit:
+            install_cmd += " -c %s" % autotest_commit
+        session_server.cmd(install_cmd, timeout=autotest_install_timeout)
+        vm_server.copy_files_from(guest_path="/tmp/install-autotest-server*log",
+                                  host_path=test.resultsdir)
+    except aexpect.ShellCmdError, e:
+        for line in e.output.splitlines():
+            logging.error(line)
+        step_failures.append(step1)
+
+    top_commit = None
+    try:
+        session_server.cmd("test -d /usr/local/autotest/.git")
+        session_server.cmd("cd /usr/local/autotest")
+        top_commit = session_server.cmd("echo `git log -n 1 --pretty=format:%H`")
+        top_commit = top_commit.strip()
+        logging.info("Autotest top commit for repo %s, branch %s: %s",
+                     autotest_repo, autotest_branch, top_commit)
+    except aexpect.ShellCmdError, e:
+        for line in e.output.splitlines():
+            logging.error(line)
+
+    if top_commit is not None:
+        session_server.close()
+        session_server = vm_server.wait_for_login(timeout=timeout,
+                                                  username='autotest',
+                                                  password='Aut0t3st')
+
+        step2 = "unittests"
+        try:
+            session_server.cmd("cd /usr/local/autotest")
+            session_server.cmd("utils/unittest_suite.py --full",
+                               timeout=unittests_run_timeout)
+        except aexpect.ShellCmdError, e:
+            for line in e.output.splitlines():
+                logging.error(line)
+            step_failures.append(step2)
+
+        step3 = "pylint"
+        try:
+            session_server.cmd("cd /usr/local/autotest")
+            session_server.cmd("utils/check_patch.py --full --yes",
+                               timeout=pylint_run_timeout)
+        except aexpect.ShellCmdError, e:
+            for line in e.output.splitlines():
+                logging.error(line)
+            step_failures.append(step3)
+
+        step4 = "client_run"
+        try:
+            session_server.cmd("cd /usr/local/autotest/client")
+            session_server.cmd("./autotest-local run sleeptest",
+                               timeout=pylint_run_timeout)
+            session_server.cmd("rm -rf results/default")
+        except aexpect.ShellCmdError, e:
+            for line in e.output.splitlines():
+                logging.error(line)
+            step_failures.append(step4)
+
+        step5 = "server_run"
+        try:
+            session_client.cmd("iptables -F")
+            session_server.cmd("cd /usr/local/autotest")
+            session_server.cmd("server/autotest-remote -m %s --ssh-user root "
+                               "--ssh-pass %s "
+                               "-c client/tests/sleeptest/control" %
+                               (client_ip, password),
+                               timeout=pylint_run_timeout)
+            session_server.cmd("rm -rf results-*")
+        except aexpect.ShellCmdError, e:
+            for line in e.output.splitlines():
+                logging.error(line)
+            step_failures.append(step5)
+
+        step6 = "registering_client_cli"
+        try:
+            label_name = "label-%s" % utils_misc.generate_random_id()
+            create_label_cmd = ("/usr/local/autotest/cli/autotest-rpc-client "
+                                "label create -t %s -w %s" %
+                                (label_name, server_ip))
+            session_server.cmd(create_label_cmd)
+
+            list_labels_cmd = ("/usr/local/autotest/cli/autotest-rpc-client "
+                               "label list -a -w %s" % server_ip)
+            list_labels_output = session_server.cmd(list_labels_cmd)
+            for line in list_labels_output.splitlines():
+                logging.debug(line)
+            if not label_name in list_labels_output:
+                raise ValueError("No label %s in the output of %s" %
+                                 (label_name, list_labels_cmd))
+
+            create_host_cmd = ("/usr/local/autotest/cli/autotest-rpc-client "
+                               "host create -t %s %s -w %s" %
+                               (label_name, client_ip, server_ip))
+            session_server.cmd(create_host_cmd)
+
+            list_hosts_cmd = ("/usr/local/autotest/cli/autotest-rpc-client "
+                               "host list -w %s" % server_ip)
+            list_hosts_output = session_server.cmd(list_hosts_cmd)
+            for line in list_hosts_output.splitlines():
+                logging.debug(line)
+            if not client_ip in list_hosts_output:
+                raise ValueError("No client %s in the output of %s" %
+                                 (client_ip, create_label_cmd))
+            if not label_name in list_hosts_output:
+                raise ValueError("No label %s in the output of %s" %
+                                 (label_name, create_label_cmd))
+
+        except (aexpect.ShellCmdError, ValueError), e:
+            if isinstance(e, aexpect.ShellCmdError):
+                for line in e.output.splitlines():
+                    logging.error(line)
+            elif isinstance(e, ValueError):
+                logging.error(e)
+            step_failures.append(step6)
+
+        step7 = "running_job_cli"
+        try:
+            session_client.cmd("iptables -F")
+
+            job_name = "Sleeptest %s" % utils_misc.generate_random_id()
+
+            def job_is_status(status):
+                list_jobs_cmd = ("/usr/local/autotest/cli/autotest-rpc-client "
+                                 "job list -a -w %s" % server_ip)
+                list_jobs_output = session_server.cmd(list_jobs_cmd)
+                if job_name in list_jobs_output:
+                    if status in list_jobs_output:
+                        return True
+                    elif "Aborted" in list_jobs_output:
+                        raise ValueError("Job is in aborted state")
+                    elif "Failed" in list_jobs_output:
+                        raise ValueError("Job is in failed state")
+                    else:
+                        return False
+                else:
+                    raise ValueError("Job %s does not show in the "
+                                     "output of %s" % list_jobs_cmd)
+
+            def job_is_completed():
+                return job_is_status("Completed")
+
+            def job_is_running():
+                return job_is_status("Running")
+
+            job_create_cmd = ("/usr/local/autotest/cli/autotest-rpc-client "
+                              "job create --test sleeptest -m %s '%s' -w %s" %
+                              (client_ip, job_name, server_ip))
+            session_server.cmd(job_create_cmd)
+
+            if not utils_misc.wait_for(job_is_running, 300, 0, 10,
+                                       "Waiting for job to start running"):
+                raise ValueError("Job did not start running")
+
+            # Wait for the session to become unresponsive
+            if not utils_misc.wait_for(lambda: not session_client.is_responsive(),
+                                       timeout=300):
+                raise error.ValueError("Client machine did not reboot")
+
+            # Establish a new client session
+            session_client = vm_client.wait_for_login(timeout=timeout)
+
+            # Wait for the job to complete
+            if not utils_misc.wait_for(job_is_completed, 300, 0, 10,
+                                       "Waiting for job to complete"):
+                raise ValueError("Job did not complete")
+
+            # Copy logs back so we can analyze them
+            vm_server.copy_files_from(guest_path="/usr/local/autotest/results/*",
+                                      host_path=test.resultsdir)
+
+        except (aexpect.ShellCmdError, ValueError), e:
+            if isinstance(e, aexpect.ShellCmdError):
+                for line in e.output.splitlines():
+                    logging.error(line)
+            elif isinstance(e, ValueError):
+                logging.error(e)
+            step_failures.append(step7)
+
+    def report_version():
+        if top_commit is not None:
+            logging.info("Autotest git repo: %s", autotest_repo)
+            logging.info("Autotest git branch: %s", autotest_repo)
+            logging.info("Autotest top commit: %s", top_commit)
+
+    if step_failures:
+        logging.error("The autotest regression testing failed")
+        report_version()
+        raise error.TestFail("The autotest regression testing had the "
+                             "following steps failed: %s" % step_failures)
+    else:
+        logging.info("The autotest regression testing passed")
+        report_version()

tests/cfg/autotest_regression.cfg

+- autotest_regression: install setup image_copy unattended_install.cdrom
+    virt_test_type = qemu libvirt
+    master_images_clone = image1
+    type = autotest_regression
+    restart_vm = yes
+    kill_vm_on_error = yes
+    login_timeout = 240
+    vms = "autotest_server_vm autotest_client_vm"
+    autotest_install_timeout = 1800
+    autotest_repo = git://github.com/autotest/autotest.git
+    autotest_branch = next
+    autotest_commit =
+