diff --git a/avocado/plugins/wrapper.py b/avocado/plugins/wrapper.py new file mode 100644 index 0000000000000000000000000000000000000000..82d46c9dcd55f69bf84010b8a5160983ae45028c --- /dev/null +++ b/avocado/plugins/wrapper.py @@ -0,0 +1,64 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See LICENSE for more details. +# +# Copyright: Red Hat Inc. 2014 +# Author: Ruda Moura + +import os +import sys + +from avocado import runtime +from avocado.core import error_codes +from avocado.core import output +from avocado.plugins import plugin + + +class Wrapper(plugin.Plugin): + + name = 'wrapper' + enabled = True + + def configure(self, parser): + self.parser = parser + wrap_group = self.parser.runner.add_argument_group( + 'Wrap avocado.utils.process module') + wrap_group.add_argument('--wrapper', action='append', default=[], + help='') + self.configured = True + + def activate(self, app_args): + view = output.View(app_args=app_args) + try: + for wrap in app_args.wrapper: + if ':' not in wrap: + if runtime.WRAP_PROCESS is None: + script = os.path.abspath(wrap) + runtime.WRAP_PROCESS = os.path.abspath(script) + else: + view.notify(event='error', + msg="You can't have multiple global" + " wrappers at once.") + sys.exit(error_codes.numeric_status['AVOCADO_CRASH']) + else: + script, cmd = wrap.split(':', 1) + script = os.path.abspath(script) + runtime.WRAP_PROCESS_NAMES_EXPR.append((script, cmd)) + if not os.path.exists(script): + view.notify(event='error', + msg="Wrapper '%s' not found!" % script) + sys.exit(error_codes.numeric_status['AVOCADO_CRASH']) + if app_args.gdb_run_bin: + view.notify(event='error', + msg='Command line option --wrapper is incompatible' + ' with option --gdb-run-bin.') + sys.exit(error_codes.numeric_status['AVOCADO_CRASH']) + except AttributeError: + pass diff --git a/avocado/runtime.py b/avocado/runtime.py index d6c521f645d636e083c980e52e60f6b97a02a544..707dfe575d1b053bd0448db7baf92f8c41f57580 100644 --- a/avocado/runtime.py +++ b/avocado/runtime.py @@ -35,6 +35,18 @@ GDB_PATH = None #: Path to the gdbserver binary GDBSERVER_PATH = None +#: The active wrapper utility script. +CURRENT_WRAPPER = None + +#: The global wrapper. +#: If set, run every process under this wrapper. +WRAP_PROCESS = None + +#: Set wrapper per program names. +#: A list of wrappers and program names. +#: Format: [ ('/path/to/wrapper.sh', 'progname'), ... ] +WRAP_PROCESS_NAMES_EXPR = [] + #: Sometimes it's useful for the framework and API to know about the test that #: is currently running, if one exists CURRENT_TEST = None diff --git a/avocado/utils/process.py b/avocado/utils/process.py index f53729468c6775afea3b99280d64e9aa55674790..6c8bc00976cea8b7f4b3f940f544fa5d6bc4c8b5 100644 --- a/avocado/utils/process.py +++ b/avocado/utils/process.py @@ -518,6 +518,25 @@ class SubProcess(object): return self.result +class WrapSubProcess(SubProcess): + + ''' + Wrap subprocess inside an utility program. + ''' + + def __init__(self, cmd, verbose=True, allow_output_check='all', + shell=False, env=None, wrapper=None): + if wrapper is None and runtime.CURRENT_WRAPPER is not None: + wrapper = runtime.CURRENT_WRAPPER + self.wrapper = wrapper + if self.wrapper: + if not os.path.exists(self.wrapper): + raise IOError("No such wrapper: '%s'" % self.wrapper) + cmd = wrapper + ' ' + cmd + super(WrapSubProcess, self).__init__(cmd, verbose, allow_output_check, + shell, env) + + class GDBSubProcess(object): ''' @@ -833,6 +852,32 @@ def should_run_inside_gdb(cmd): return False +def should_run_inside_wrapper(cmd): + ''' + Wether the given command should be run inside the wrapper utility. + + :param cmd: the command arguments, from where we extract the binary name + ''' + runtime.CURRENT_WRAPPER = None + args = shlex.split(cmd) + cmd_binary_name = args[0] + + for script, cmd in runtime.WRAP_PROCESS_NAMES_EXPR: + if os.path.isabs(cmd_binary_name) and os.path.isabs(cmd) is False: + cmd_binary_name = os.path.basename(cmd_binary_name) + cmd = os.path.basename(cmd) + if cmd_binary_name == cmd: + runtime.CURRENT_WRAPPER = script + + if runtime.WRAP_PROCESS is not None and runtime.CURRENT_WRAPPER is None: + runtime.CURRENT_WRAPPER = runtime.WRAP_PROCESS + + if runtime.CURRENT_WRAPPER is None: + return False + else: + return True + + def get_sub_process_klass(cmd): ''' Which sub process implementation should be used @@ -843,6 +888,8 @@ def get_sub_process_klass(cmd): ''' if should_run_inside_gdb(cmd): return GDBSubProcess + elif should_run_inside_wrapper(cmd): + return WrapSubProcess else: return SubProcess diff --git a/docs/source/WrapProcess.rst b/docs/source/WrapProcess.rst new file mode 100644 index 0000000000000000000000000000000000000000..f6c4734cb30355b0a364f1d348e3574804a08882 --- /dev/null +++ b/docs/source/WrapProcess.rst @@ -0,0 +1,61 @@ +Wrap process in tests +===================== + +Avocado allows the instrumentation of applications being +run by a test in a transparent way. + +The user specifies a script ("the wrapper") to be used to run the actual +program called by the test. If the instrument is +implemented correctly, it should not interfere with the test behavior. + +So it means that the wrapper should avoid to change the return status, +standard output and standard error messages of the process. + +The user can optionally specify a target program to wrap. + +Usage +----- + +This feature is implemented as a plugin, that adds the `--wrapper` option +to the Avocado `run` command. For a detailed explanation please consult the +avocado man page. + +Example of a transparent way of running strace as a wrapper:: + + #!/bin/sh + exec strace -ff -o $AVOCADO_TEST_LOGDIR/strace.log -- $@ + + +Now you can run:: + + # run all programs started by test.py with ~/bin/my-wrapper.sh + $ scripts/avocado run --wrapper ~/bin/my-wrapper.sh tests/test.py + + # run only my-binary (if/when started by a test) with ~/bin/my-wrapper.sh + $ scripts/avocado run --wrapper ~/bin/my-wrapper.sh:my-binary tests/test.py + + +Caveats +------- + +* It is not possible to debug with GDB (`--gdb-run-bin`) and use + wrappers (`--wrapper`), both options together are incompatible. + +* You cannot set multiples (global) wrappers + -- like `--wrapper foo.sh --wrapper bar.sh` -- it will trigger an error. + You should use a single script that performs both things + you are trying to achieve. + +* The only process that can be wrapper are those that uses the Avocado + module `avocado.utils.process` and the modules that make use of it, + like `avocado.utils.build` and so on. + +* If paths are not absolute, then the process name matches with the base name, + so `--wrapper foo.sh:make` will match `/usr/bin/make`, `/opt/bin/make` + and `/long/path/to/make`. + +* When you use a relative path to a script, it will use the current path + of the running avocado program. + Example: If I'm running avocado on `/home/user/project/avocado`, + then `avocado run --wrapper examples/wrappers/strace.sh datadir` will + set the wrapper to `/home/user/project/avocado/examples/wrappers/strace.sh` diff --git a/docs/source/index.rst b/docs/source/index.rst index 35c8a20384873b09c090178619dcd41ec36ccdfa..1bf7ceb828c3bada64f39b2329d5ff7b7c40d389 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -23,6 +23,7 @@ Contents: VirtualMachinePlugin RemoteMachinePlugin DebuggingWithGDB + WrapProcess ContributionGuide api/modules diff --git a/examples/wrappers/dummy.sh b/examples/wrappers/dummy.sh new file mode 100755 index 0000000000000000000000000000000000000000..1a96da4dfdc9a54ba0766381f80da373a9ce4df1 --- /dev/null +++ b/examples/wrappers/dummy.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See LICENSE for more details. +# +# Copyright: Red Hat Inc. 2014 +# Author: Ruda Moura + +exec -- $@ diff --git a/examples/wrappers/ltrace.sh b/examples/wrappers/ltrace.sh new file mode 100755 index 0000000000000000000000000000000000000000..f390b084fe8a8510749e2787392dca26b6c9b968 --- /dev/null +++ b/examples/wrappers/ltrace.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See LICENSE for more details. +# +# Copyright: Red Hat Inc. 2014 +# Author: Ruda Moura + +# Map interesting signals to exit codes (see kill -L) +# Example: SIGHUP (kill -1) 128+1 = 129 + +declare -A signal_map +signal_map[SIGHUP]=129 +signal_map[SIGINT]=130 +signal_map[SIGQUIT]=131 +signal_map[SIGILL]=132 +signal_map[SIGTRAP]=133 +signal_map[SIGABRT]=134 +signal_map[SIGBUS]=135 +signal_map[SIGFPE]=136 +signal_map[SIGKILL]=137 +signal_map[SIGUSR1]=138 +signal_map[SIGSEGV]=139 +signal_map[SIGUSR2]=140 +signal_map[SIGPIPE]=141 +signal_map[SIGALRM]=142 +signal_map[SIGTERM]=143 +signal_map[SIGSTKFLT]=144 +signal_map[SIGSTKFLT]=144 +signal_map[SIGXCPU]=152 +signal_map[SIGXFSZ]=153 +signal_map[SIGVTALRM]=154 +signal_map[SIGPROF]=155 +signal_map[SIGIO]=157 +signal_map[SIGPWR]=158 +signal_map[SIGSYS]=159 +signal_map[UNKNOWN_SIGNAL]=160 + +ltrace -f -o $AVOCADO_TEST_LOGDIR/ltrace.log.$$ -- $@ + +signal_name=$(sed -ne 's/^.*+++ killed by \([A-Z_]\+\) +++$/\1/p' $AVOCADO_TEST_LOGDIR/ltrace.log.$$) +if [ -n "$signal_name" ] ; then + exit ${signal_map[$signal_name]} +fi + +exit_status=$(sed -ne 's/^.*+++ exited (status \([0-9]\+\)) +++$/\1/p' $AVOCADO_TEST_LOGDIR/ltrace.log.$$) +if [ -n "$exit_status" ] ; then + exit $exit_status +fi + +exit 0 diff --git a/examples/wrappers/perf.sh b/examples/wrappers/perf.sh new file mode 100755 index 0000000000000000000000000000000000000000..64856a99f0f00b80f50e10119095b30c33e8a637 --- /dev/null +++ b/examples/wrappers/perf.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See LICENSE for more details. +# +# Copyright: Red Hat Inc. 2014 +# Author: Ruda Moura + +exec perf record -o $AVOCADO_TEST_LOGDIR/perf.data.$$ -- $@ diff --git a/examples/wrappers/strace.sh b/examples/wrappers/strace.sh new file mode 100755 index 0000000000000000000000000000000000000000..b1e162a84fce8ad1ed4682acce175056b43d6afa --- /dev/null +++ b/examples/wrappers/strace.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See LICENSE for more details. +# +# Copyright: Red Hat Inc. 2014 +# Author: Ademar de Souza Reis Jr + +exec strace -ff -o $AVOCADO_TEST_LOGDIR/strace.log -- $@ diff --git a/examples/wrappers/valgrind.sh b/examples/wrappers/valgrind.sh new file mode 100755 index 0000000000000000000000000000000000000000..62a11a285bd346b1cf754a4f206f5997a8ca0e99 --- /dev/null +++ b/examples/wrappers/valgrind.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See LICENSE for more details. +# +# Copyright: Red Hat Inc. 2014 +# Author: Ruda Moura + +valgrind \ + --tool=memcheck \ + --verbose \ + --trace-children=yes \ + --leak-check=full \ + --log-file=$AVOCADO_TEST_LOGDIR/valgrind.log.$$ -- $@ diff --git a/man/avocado.rst b/man/avocado.rst index 982a3010dc1ba30b91eb2408102d6f2c4d01e59a..cb7466d36370394ff57fed9fd895712c3b77e14b 100644 --- a/man/avocado.rst +++ b/man/avocado.rst @@ -310,6 +310,46 @@ In this example, `/tmp/disable-signals` is a simple text file containing two lin Each line is a GDB command, so you can have from simple to very complex debugging environments configured like that. +WRAP PROCESS IN TESTS +===================== + +Avocado allows the instrumentation of applications being +run by a test in a transparent way. + +The user specify a script ("the wrapper") to be used to run the actual +program called by the test. If the instrument is +implemented correctly, it should not interfere with the test behavior. + +So it means that the wrapper should avoid to change the return status, +standard output and standard error messages of the process. + +By using an optional parameter to the wrapper, you can specify the +"target binary" to wrap, so that for every program spawned by the test, +the program name will be compared to the target binary. + +If the target binary is absolute path and the program name is absolute, +then both paths should be equal to the wrapper take effect, otherwise +the wrapper will not be used. + +For the case that the target binary is not absolute or the program name +is not absolute, then both will be compared by its base name, ignoring paths. + +Examples:: + + $ avocado run datadir --wrapper examples/wrappers/strace.sh + $ avocado run datadir --wrapper examples/wrappers/ltrace.sh:make \ + --wrapper examples/wrappers/perf.sh:datadir + +Note that it's not possible to use ``--gdb-run-bin`` together +with ``--wrapper``, they are incompatible.:: + + $ avocado run mytest --wrapper examples/wrappers/strace:/opt/bin/foo + +In this case, the possible program that can wrapped by ``mytest`` is +``/opt/bin/foo`` (absolute paths equal) and ``foo`` without absolute path +will be wrapped too, but ``/opt/bin/foo`` will never be wrapped, because +the absolute paths are not equal. + RECORDING TEST REFERENCE OUTPUT =============================== diff --git a/selftests/all/functional/avocado/wrapper_tests.py b/selftests/all/functional/avocado/wrapper_tests.py new file mode 100644 index 0000000000000000000000000000000000000000..53028a56a36714db877d137d3e80190cba172d54 --- /dev/null +++ b/selftests/all/functional/avocado/wrapper_tests.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See LICENSE for more details. +# +# Copyright: Red Hat Inc. 2014 +# Author: Ruda Moura + +import os +import sys +import unittest +import tempfile + +# simple magic for using scripts within a source tree +basedir = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', '..', '..') +basedir = os.path.abspath(basedir) +if os.path.isdir(os.path.join(basedir, 'avocado')): + sys.path.append(basedir) + +from avocado.utils import process +from avocado.utils import script + +SCRIPT_CONTENT = """#!/bin/bash +touch %s +exec -- $@ +""" + +DUMMY_CONTENT = """#!/bin/bash +exec -- $@ +""" + + +class WrapperTest(unittest.TestCase): + + def setUp(self): + self.tmpfile = tempfile.mktemp() + self.script = script.TemporaryScript( + 'success.sh', + SCRIPT_CONTENT % self.tmpfile, + 'avocado_wrapper_functional') + self.script.save() + self.dummy = script.TemporaryScript( + 'dummy.sh', + DUMMY_CONTENT, + 'avocado_wrapper_functional') + self.dummy.save() + + def test_global_wrapper(self): + os.chdir(basedir) + cmd_line = './scripts/avocado run --wrapper %s examples/tests/datadir.py' % self.script.path + result = process.run(cmd_line, ignore_status=True) + expected_rc = 0 + self.assertEqual(result.exit_status, expected_rc, + "Avocado did not return rc %d:\n%s" % + (expected_rc, result)) + self.assertTrue(os.path.exists(self.tmpfile), + "Wrapper did not create file %s" % self.tmpfile) + + def test_process_wrapper(self): + os.chdir(basedir) + cmd_line = './scripts/avocado run --wrapper %s:datadir examples/tests/datadir.py' % self.script.path + result = process.run(cmd_line, ignore_status=True) + expected_rc = 0 + self.assertEqual(result.exit_status, expected_rc, + "Avocado did not return rc %d:\n%s" % + (expected_rc, result)) + self.assertTrue(os.path.exists(self.tmpfile), + "Wrapper did not create file %s" % self.tmpfile) + + def test_both_wrappers(self): + os.chdir(basedir) + cmd_line = './scripts/avocado run --wrapper %s --wrapper %s:datadir examples/tests/datadir.py' % (self.dummy.path, self.script.path) + result = process.run(cmd_line, ignore_status=True) + expected_rc = 0 + self.assertEqual(result.exit_status, expected_rc, + "Avocado did not return rc %d:\n%s" % + (expected_rc, result)) + self.assertTrue(os.path.exists(self.tmpfile), + "Wrapper did not create file %s" % self.tmpfile) + + def tearDown(self): + self.script.remove() + self.dummy.remove() + try: + os.remove(self.tmpfile) + except OSError: + pass + + +if __name__ == '__main__': + unittest.main()