From d388faae3f1512d8bed268320986ada888bfadde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Doktor?= Date: Sat, 23 Jul 2016 11:50:47 +0200 Subject: [PATCH] avocado.plugins: Add plugin to run job in docker container MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Similarly to --remote or --vm plugins this plugin allows one to run the job inside a docker container by specifying the docker image. It executes a new container, then attaches it and uses it similarly as --remote plugin uses remote machine. To check it out you can use "ldoktor/fedora-avocado" image which is available on (the default) hub.docker.com Signed-off-by: Cleber Rosa Signed-off-by: Lukáš Doktor --- avocado/core/remote/runner.py | 2 + avocado/plugins/docker.py | 177 +++++++++++++++++++++++++++ docs/source/RunningTestsRemotely.rst | 47 +++++++ setup.py | 1 + 4 files changed, 227 insertions(+) create mode 100644 avocado/plugins/docker.py diff --git a/avocado/core/remote/runner.py b/avocado/core/remote/runner.py index 7b4cd845..54e489b9 100644 --- a/avocado/core/remote/runner.py +++ b/avocado/core/remote/runner.py @@ -203,6 +203,7 @@ class RemoteTestRunner(TestRunner): fabric_debugfile = os.path.join(self.job.logdir, 'remote.log') paramiko_logger = logging.getLogger('paramiko') fabric_logger = logging.getLogger('avocado.fabric') + remote_logger = logging.getLogger('avocado.remote') app_logger = logging.getLogger('avocado.debug') fmt = ('%(asctime)s %(module)-10.10s L%(lineno)-.4d %(' 'levelname)-5.5s| %(message)s') @@ -211,6 +212,7 @@ class RemoteTestRunner(TestRunner): file_handler.setFormatter(formatter) fabric_logger.addHandler(file_handler) paramiko_logger.addHandler(file_handler) + remote_logger.addHandler(file_handler) logger_list = [fabric_logger] if self.job.args.show_job_log: logger_list.append(app_logger) diff --git a/avocado/plugins/docker.py b/avocado/plugins/docker.py new file mode 100644 index 00000000..b27c0bb6 --- /dev/null +++ b/avocado/plugins/docker.py @@ -0,0 +1,177 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See LICENSE for more details. +# +# Copyright: 2016 Red Hat, Inc. +# Author: Lukas Doktor +"""Run the job inside a docker container.""" + +import logging +import time + +import aexpect +from avocado.core.plugin_interfaces import CLI +from avocado.core.remote.runner import RemoteTestRunner +from avocado.utils import process +from avocado.utils.wait import wait_for + + +LOG = logging.getLogger('avocado.remote') + + +class DockerRemoter(object): + + """ + Remoter object similar to `avocado.core.remoter` which implements subset + of the commands on docker container. + """ + + def __init__(self, dkrcmd, image): + """ + Executes docker container and attaches it. + + :param dkrcmd: The base docker binary (or command) + :param image: docker image to be used in this instance + """ + self._dkrcmd = dkrcmd + run_cmd = "%s run -t -i -d '%s' bash" % (self._dkrcmd, image) + self._docker_id = process.system_output(run_cmd, 10).strip() + self._docker = aexpect.ShellSession("%s attach %s" + % (self._dkrcmd, self._docker_id)) + # Disable echo to avoid duplicate output + self._docker.cmd("stty -echo") + + def get_cid(self): + """ Return this remoter's container ID """ + return self._docker_id + + def makedir(self, remote_path): + """ + Create a directory on the container + + :warning: No other process must be running on foreground + :param remote_path: the remote path to create. + """ + self._docker.cmd("mkdir -p %s" % remote_path) + + def send_files(self, local_path, remote_path): + """ + Send files to the container + """ + process.run("%s cp %s %s:%s" % (self._dkrcmd, local_path, + self._docker_id, remote_path)) + + def receive_files(self, local_path, remote_path): + """ + Receive files from the container + """ + process.run("%s cp %s:%s %s" % (self._dkrcmd, self._docker_id, + remote_path, local_path)) + + def run(self, command, ignore_status=False, quiet=None, timeout=60): + """ + Run command inside the container + """ + def print_func(*args, **kwargs): # pylint: disable=W0613 + """ Accept anything and does nothing """ + pass + if timeout is None: + timeout = 31536000 # aexpect does not support None, use one year + start = time.time() + if quiet is not False: + print_func = LOG.debug + status, output = self._docker.cmd_status_output(command, + timeout=timeout, + print_func=print_func) + result = process.CmdResult(command, output, '', status, + time.time() - start) + if status and not ignore_status: + raise process.CmdError(command, result, "in container %s" + % self._docker_id) + return result + + def cleanup(self): + """ + Stop the container and remove it + """ + process.system("%s stop -t 1 %s" % (self._dkrcmd, self._docker_id)) + process.system("%s rm %s" % (self._dkrcmd, self._docker_id)) + + def close(self): + """ + Safely postprocess the container + + :note: It won't remove the container, you need to do it manually + """ + self._docker.sendline("exit") + # Leave the process up to 10s to finish, then nuke it + wait_for(lambda: not self._docker.is_alive(), 10) + self._docker.close() + + +class DockerTestRunner(RemoteTestRunner): + + """ + Test runner which runs the job inside a docker container + """ + + remote_test_dir = "/avocado_remote_test_dir" # Absolute path only + + def __init__(self, job, test_result): + super(DockerTestRunner, self).__init__(job, test_result) + self.remote = None # Will be set in `setup` + + def setup(self): + dkrcmd = self.job.args.docker_cmd + self.remote = DockerRemoter(dkrcmd, self.job.args.docker) + # We need to create the base dir, otherwise docker creates it as root + self.remote.makedir(self.remote_test_dir) + self.job.log.info("DOCKER : Container id '%s'" + % self.remote.get_cid()) + self.job.args.remote_no_copy = self.job.args.docker_no_copy + + def tear_down(self): + self.remote.close() + if not self.job.args.docker_no_cleanup: + self.remote.cleanup() + + +class Docker(CLI): + + """ + Run the job inside a docker container + """ + + name = 'docker' + description = "Run tests inside docker container" + + def configure(self, parser): + run_subcommand_parser = parser.subcommands.choices.get('run', None) + if run_subcommand_parser is None: + return + + msg = 'test execution inside docker container' + cmd_parser = run_subcommand_parser.add_argument_group(msg) + cmd_parser.add_argument("--docker", help="Name of the docker image to" + "run tests on.", metavar="IMAGE") + cmd_parser.add_argument("--docker-cmd", default="docker", + help="Override the docker command, eg. 'sudo " + "docker' or other base docker options like " + "hypervisor. Default: '%(default)s'", + metavar="CMD") + cmd_parser.add_argument("--docker-no-copy", action="store_true", + help="Assume tests are already in the " + "container") + cmd_parser.add_argument("--docker-no-cleanup", action="store_true", + help="Preserve container after test") + + def run(self, args): + if getattr(args, "docker", None): + args.test_runner = DockerTestRunner diff --git a/docs/source/RunningTestsRemotely.rst b/docs/source/RunningTestsRemotely.rst index d535c8ab..3888b17c 100644 --- a/docs/source/RunningTestsRemotely.rst +++ b/docs/source/RunningTestsRemotely.rst @@ -147,6 +147,53 @@ execute them. A bit of extra logging information is added to your job summary, mainly to distinguish the regular execution from the remote one. Note here that we did not need `--vm-password` because the SSH key is already setup. +Running Tests on a Docker container +=================================== + +Avocado also lets you run tests on a Docker container, starting and +cleaning it up automatically with every execution. + +You can check if this feature (a plugin) is enabled by running:: + + $ avocado plugins + ... + docker Run tests inside docker container + ... + +Docker container images +----------------------- + +Avocado needs to be present inside the container image in order for +the test execution to be properly performed. There's one ready to use +image (``ldoktor/fedora-avocado``) in the default image repository +(``docker.io``):: + + $ docker pull ldoktor/fedora-avocado + Using default tag: latest + Trying to pull repository docker.io/ldoktor/fedora-avocado ... + latest: Pulling from docker.io/ldoktor/fedora-avocado + ... + Status: Downloaded newer image for docker.io/ldoktor/fedora-avocado:latest + +Running your test +----------------- + +Assuming your system is properly setup to run Docker, including having +an image with Avocado, you can run a test inside the container with a +command similar to:: + + $ avocado run passtest.py warntest.py failtest.py --docker ldoktor/fedora-avocado + DOCKER : Container id '4bcbcd69801211501a0dde5926c0282a9630adbe29ecb17a21ef04f024366943' + JOB ID : db309f5daba562235834f97cad5f4458e3fe6e32 + JOB LOG : $HOME/avocado/job-results/job-2016-07-25T08.01-db309f5/job.log + TESTS : 3 + (1/3) /avocado_remote_test_dir/$HOME/passtest.py:PassTest.test: PASS (0.00 s) + (2/3) /avocado_remote_test_dir/$HOME/warntest.py:WarnTest.test: WARN (0.00 s) + (3/3) /avocado_remote_test_dir/$HOME/failtest.py:FailTest.test: FAIL (0.00 s) + RESULTS : PASS 1 | ERROR 0 | FAIL 1 | SKIP 0 | WARN 1 | INTERRUPT 0 + JOB HTML : $HOME/avocado/job-results/job-2016-07-25T08.01-db309f5/html/results.html + TESTS TIME : 0.00 s + Environment Variables ===================== diff --git a/setup.py b/setup.py index 4a7c6211..067d0b40 100755 --- a/setup.py +++ b/setup.py @@ -139,6 +139,7 @@ if __name__ == '__main__': 'replay = avocado.plugins.replay:Replay', 'tap = avocado.plugins.tap:TAP', 'vm = avocado.plugins.vm:VM', + 'docker = avocado.plugins.docker:Docker', ], 'avocado.plugins.cli.cmd': [ 'config = avocado.plugins.config:Config', -- GitLab