diff --git a/avocado/core/defaults.py b/avocado/core/defaults.py new file mode 100644 index 0000000000000000000000000000000000000000..b84eeb7512d14dd10eb54d061d6ce2201a32cf64 --- /dev/null +++ b/avocado/core/defaults.py @@ -0,0 +1,20 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See LICENSE for more details. +# +# Copyright: Red Hat Inc. 2018 +# Author: Cleber Rosa + +""" +The Avocado core defaults +""" + +#: The encoding used by default on all data input +ENCODING = 'utf-8' diff --git a/avocado/core/test.py b/avocado/core/test.py index 8fc6beb92bde005af8ac843dfd6840c21a0ef273..e50d4dec5f16479ae7a2479cd674e4dd963853ad 100644 --- a/avocado/core/test.py +++ b/avocado/core/test.py @@ -33,6 +33,7 @@ from difflib import unified_diff from six import string_types, iteritems from . import data_dir +from . import defaults from . import exceptions from . import output from . import parameters @@ -1138,7 +1139,7 @@ class SimpleTest(Test): # process.run uses shlex.split(), the self.path needs to be escaped result = process.run(self._command, verbose=True, - env=test_params) + env=test_params, encoding=defaults.ENCODING) self._log_detailed_cmd_info(result) except process.CmdError as details: diff --git a/avocado/utils/process.py b/avocado/utils/process.py index 1b1038d5377c7ca27f5e4e36939aa3de38e1d437..399ed1defca1b264238ecb9f567658eb3896bafc 100644 --- a/avocado/utils/process.py +++ b/avocado/utils/process.py @@ -32,7 +32,7 @@ import threading import time from io import BytesIO, UnsupportedOperation -from six import string_types +from six import PY2, string_types from . import gdb from . import runtime @@ -99,10 +99,34 @@ class CmdError(Exception): return "CmdError" -def can_sudo(cmd=None): +def normalize_cmd(cmd, encoding=None): + """ + Normalize a command to be safe for :func:`shlex.split` + + :param cmd: the command line to be passed to :func:`shlex.split` + :type cmd: str or bytes + :param encoding: the encoding to use for encode/decode operations + :type encoding: str + """ + if encoding is None: + encoding = sys.getdefaultencoding() + if PY2: + if not isinstance(cmd, str): + cmd = cmd.encode(encoding) + else: + if isinstance(cmd, bytes): + cmd = cmd.decode(encoding) + return cmd + + +def can_sudo(cmd=None, encoding=None): """ Check whether sudo is available (or running as root) """ + if cmd is not None: + if encoding is None: + encoding = sys.getdefaultencoding() + cmd = normalize_cmd(cmd, encoding) if os.getuid() == 0: # Root return True @@ -233,7 +257,7 @@ def get_children_pids(ppid, recursive=False): return children -def binary_from_shell_cmd(cmd): +def binary_from_shell_cmd(cmd, encoding=None): """ Tries to find the first binary path from a simple shell-like command. @@ -242,6 +266,9 @@ def binary_from_shell_cmd(cmd): :param cmd: simple shell-like binary :return: first found binary from the cmd """ + if encoding is None: + encoding = sys.getdefaultencoding() + cmd = normalize_cmd(cmd, encoding) try: cmds = shlex.split(cmd) except ValueError: @@ -437,7 +464,7 @@ class SubProcess(object): def __init__(self, cmd, verbose=True, allow_output_check=None, shell=False, env=None, sudo=False, - ignore_bg_processes=False): + ignore_bg_processes=False, encoding=None): """ Creates the subprocess object, stdout/err, reader threads and locks. @@ -481,8 +508,16 @@ class SubProcess(object): in missing output produced by those daemons after the main thread finishes and also it allows those daemons to be running after the process finishes. + :param encoding: the encoding to use for the text representation + of the command result stdout and stderr, with the + default being Python's own, that is, + (:func:`sys.getdefaultencoding`). + :type encoding: str :raises: ValueError if incorrect values are given to parameters """ + if encoding is None: + encoding = sys.getdefaultencoding() + cmd = normalize_cmd(cmd, encoding) if sudo: self.cmd = self._prepend_sudo(cmd, shell) else: @@ -498,7 +533,7 @@ class SubProcess(object): allow_output_check) raise ValueError(msg) self.allow_output_check = allow_output_check - self.result = CmdResult(self.cmd) + self.result = CmdResult(self.cmd, encoding=encoding) self.shell = shell if env: self.env = os.environ.copy() @@ -812,7 +847,7 @@ class WrapSubProcess(SubProcess): def __init__(self, cmd, verbose=True, allow_output_check=None, shell=False, env=None, wrapper=None, sudo=False, - ignore_bg_processes=False): + ignore_bg_processes=False, encoding=None): if wrapper is None and CURRENT_WRAPPER is not None: wrapper = CURRENT_WRAPPER self.wrapper = wrapper @@ -822,7 +857,7 @@ class WrapSubProcess(SubProcess): cmd = wrapper + ' ' + cmd super(WrapSubProcess, self).__init__(cmd, verbose, allow_output_check, shell, env, sudo, - ignore_bg_processes) + ignore_bg_processes, encoding) class GDBSubProcess(object): @@ -833,7 +868,7 @@ class GDBSubProcess(object): def __init__(self, cmd, verbose=True, allow_output_check=None, shell=False, - env=None, sudo=False, ignore_bg_processes=False): + env=None, sudo=False, ignore_bg_processes=False, encoding=None): """ Creates the subprocess object, stdout/err, reader threads and locks. @@ -859,13 +894,15 @@ class GDBSubProcess(object): implementation, since the GDB wrapping code does not have support to run commands in that way. """ - + if encoding is None: + encoding = sys.getdefaultencoding() + cmd = normalize_cmd(cmd, encoding) self.cmd = cmd self.args = shlex.split(cmd) self.binary = self.args[0] self.binary_path = os.path.abspath(self.cmd) - self.result = CmdResult(cmd) + self.result = CmdResult(cmd, encoding=encoding) self.gdb_server = gdb.GDBServer(gdb.GDBSERVER_PATH) self.gdb = gdb.GDB(gdb.GDB_PATH) @@ -1185,7 +1222,7 @@ def should_run_inside_wrapper(cmd): return True -def get_sub_process_klass(cmd): +def get_sub_process_klass(cmd, encoding=None): """ Which sub process implementation should be used @@ -1193,6 +1230,9 @@ def get_sub_process_klass(cmd): :param cmd: the command arguments, from where we extract the binary name """ + if encoding is None: + encoding = sys.getdefaultencoding() + cmd = normalize_cmd(cmd, encoding) if should_run_inside_gdb(cmd): return GDBSubProcess elif should_run_inside_wrapper(cmd): @@ -1203,7 +1243,8 @@ def get_sub_process_klass(cmd): def run(cmd, timeout=None, verbose=True, ignore_status=False, allow_output_check=None, shell=False, - env=None, sudo=False, ignore_bg_processes=False): + env=None, sudo=False, ignore_bg_processes=False, + encoding=None): """ Run a subprocess, returning a CmdResult object. @@ -1247,14 +1288,22 @@ def run(cmd, timeout=None, verbose=True, ignore_status=False, has a sudo configuration such that a password won't be prompted. If that's not the case, the command will straight out fail. + :param encoding: the encoding to use for the text representation + of the command result stdout and stderr, with the + default being Python's own, that is, + (:func:`sys.getdefaultencoding`). + :type encoding: str :return: An :class:`CmdResult` object. :raise: :class:`CmdError`, if ``ignore_status=False``. """ - klass = get_sub_process_klass(cmd) + if encoding is None: + encoding = sys.getdefaultencoding() + klass = get_sub_process_klass(cmd, encoding) sp = klass(cmd=cmd, verbose=verbose, allow_output_check=allow_output_check, shell=shell, env=env, - sudo=sudo, ignore_bg_processes=ignore_bg_processes) + sudo=sudo, ignore_bg_processes=ignore_bg_processes, + encoding=encoding) cmd_result = sp.run(timeout=timeout) fail_condition = cmd_result.exit_status != 0 or cmd_result.interrupted if fail_condition and not ignore_status: @@ -1264,7 +1313,8 @@ def run(cmd, timeout=None, verbose=True, ignore_status=False, def system(cmd, timeout=None, verbose=True, ignore_status=False, allow_output_check=None, shell=False, - env=None, sudo=False, ignore_bg_processes=False): + env=None, sudo=False, ignore_bg_processes=False, + encoding=None): """ Run a subprocess, returning its exit code. @@ -1308,6 +1358,11 @@ def system(cmd, timeout=None, verbose=True, ignore_status=False, has a sudo configuration such that a password won't be prompted. If that's not the case, the command will straight out fail. + :param encoding: the encoding to use for the text representation + of the command result stdout and stderr, with the + default being Python's own, that is, + (:func:`sys.getdefaultencoding`). + :type encoding: str :return: Exit code. :rtype: int @@ -1315,14 +1370,15 @@ def system(cmd, timeout=None, verbose=True, ignore_status=False, """ cmd_result = run(cmd=cmd, timeout=timeout, verbose=verbose, ignore_status=ignore_status, allow_output_check=allow_output_check, shell=shell, env=env, - sudo=sudo, ignore_bg_processes=ignore_bg_processes) + sudo=sudo, ignore_bg_processes=ignore_bg_processes, + encoding=encoding) return cmd_result.exit_status def system_output(cmd, timeout=None, verbose=True, ignore_status=False, allow_output_check=None, shell=False, env=None, sudo=False, ignore_bg_processes=False, - strip_trail_nl=True): + strip_trail_nl=True, encoding=None): """ Run a subprocess, returning its output. @@ -1370,6 +1426,11 @@ def system_output(cmd, timeout=None, verbose=True, ignore_status=False, :type ignore_bg_processes: bool :param strip_trail_nl: Whether to strip the trailing newline :type strip_trail_nl: bool + :param encoding: the encoding to use for the text representation + of the command result stdout and stderr, with the + default being Python's own, that is, + (:func:`sys.getdefaultencoding`). + :type encoding: str :return: Command output. :rtype: bytes @@ -1377,7 +1438,8 @@ def system_output(cmd, timeout=None, verbose=True, ignore_status=False, """ cmd_result = run(cmd=cmd, timeout=timeout, verbose=verbose, ignore_status=ignore_status, allow_output_check=allow_output_check, shell=shell, env=env, - sudo=sudo, ignore_bg_processes=ignore_bg_processes) + sudo=sudo, ignore_bg_processes=ignore_bg_processes, + encoding=encoding) if strip_trail_nl: return cmd_result.stdout.rstrip(b'\n\r') return cmd_result.stdout diff --git a/selftests/unit/test_utils_process.py b/selftests/unit/test_utils_process.py index 1ca8da18c76736ca07709925d231823bd2eb7cf0..dc8a5a26bf29e95301ce8e6498359c98ca7fda69 100644 --- a/selftests/unit/test_utils_process.py +++ b/selftests/unit/test_utils_process.py @@ -16,7 +16,15 @@ from avocado.utils import path from six import string_types -TRUE_CMD = path.find_command('true') +def probe_binary(binary): + try: + return path.find_command(binary) + except path.CmdNotFoundError: + return None + + +TRUE_CMD = probe_binary('true') +ECHO_CMD = probe_binary('echo') class TestSubProcess(unittest.TestCase): @@ -57,6 +65,8 @@ class TestGDBProcess(unittest.TestCase): self.assertFalse(process.should_run_inside_gdb("foo bar baz")) self.assertFalse(process.should_run_inside_gdb("foo ' ")) + @unittest.skipUnless(TRUE_CMD, + '"true" binary not available') def test_get_sub_process_klass(self): gdb.GDB_RUN_BINARY_NAMES_EXPR = [] self.assertIs(process.get_sub_process_klass(TRUE_CMD), @@ -92,6 +102,8 @@ def mock_fail_find_cmd(cmd, default=None): class TestProcessRun(unittest.TestCase): + @unittest.skipUnless(TRUE_CMD, + '"true" binary not available') @mock.patch.object(path, 'find_command', mock.Mock(return_value=TRUE_CMD)) @mock.patch.object(os, 'getuid', @@ -101,6 +113,8 @@ class TestProcessRun(unittest.TestCase): p = process.SubProcess(cmd='ls -l') self.assertEqual(p.cmd, expected_command) + @unittest.skipUnless(TRUE_CMD, + '"true" binary not available') @mock.patch.object(path, 'find_command', mock.Mock(return_value=TRUE_CMD)) @mock.patch.object(os, 'getuid', mock.Mock(return_value=0)) @@ -109,6 +123,8 @@ class TestProcessRun(unittest.TestCase): p = process.SubProcess(cmd='ls -l') self.assertEqual(p.cmd, expected_command) + @unittest.skipUnless(TRUE_CMD, + '"true" binary not available') @mock.patch.object(path, 'find_command', mock.Mock(return_value=TRUE_CMD)) @mock.patch.object(os, 'getuid', @@ -125,6 +141,8 @@ class TestProcessRun(unittest.TestCase): p = process.SubProcess(cmd='ls -l', sudo=True) self.assertEqual(p.cmd, expected_command) + @unittest.skipUnless(TRUE_CMD, + '"true" binary not available') @mock.patch.object(path, 'find_command', mock.Mock(return_value=TRUE_CMD)) @mock.patch.object(os, 'getuid', mock.Mock(return_value=0)) @@ -133,6 +151,8 @@ class TestProcessRun(unittest.TestCase): p = process.SubProcess(cmd='ls -l', sudo=True) self.assertEqual(p.cmd, expected_command) + @unittest.skipUnless(TRUE_CMD, + '"true" binary not available') @mock.patch.object(path, 'find_command', mock.Mock(return_value=TRUE_CMD)) @mock.patch.object(os, 'getuid', mock.Mock(return_value=1000)) @@ -148,6 +168,8 @@ class TestProcessRun(unittest.TestCase): p = process.SubProcess(cmd='ls -l', sudo=True, shell=True) self.assertEqual(p.cmd, expected_command) + @unittest.skipUnless(TRUE_CMD, + '"true" binary not available') @mock.patch.object(path, 'find_command', mock.Mock(return_value=TRUE_CMD)) @mock.patch.object(os, 'getuid', mock.Mock(return_value=0)) @@ -156,6 +178,8 @@ class TestProcessRun(unittest.TestCase): p = process.SubProcess(cmd='ls -l', sudo=True, shell=True) self.assertEqual(p.cmd, expected_command) + @unittest.skipUnless(TRUE_CMD, + '"true" binary not available') @mock.patch.object(path, 'find_command', mock.Mock(return_value=TRUE_CMD)) @mock.patch.object(os, 'getuid', mock.Mock(return_value=1000)) @@ -164,6 +188,8 @@ class TestProcessRun(unittest.TestCase): p = process.run(cmd='ls -l', ignore_status=True) self.assertEqual(p.command, expected_command) + @unittest.skipUnless(TRUE_CMD, + '"true" binary not available') @mock.patch.object(path, 'find_command', mock.Mock(return_value=TRUE_CMD)) @mock.patch.object(os, 'getuid', mock.Mock(return_value=0)) @@ -172,6 +198,8 @@ class TestProcessRun(unittest.TestCase): p = process.run(cmd='ls -l', ignore_status=True) self.assertEqual(p.command, expected_command) + @unittest.skipUnless(TRUE_CMD, + '"true" binary not available') @mock.patch.object(path, 'find_command', mock.Mock(return_value=TRUE_CMD)) @mock.patch.object(os, 'getuid', mock.Mock(return_value=1000)) @@ -187,6 +215,8 @@ class TestProcessRun(unittest.TestCase): p = process.run(cmd='ls -l', sudo=True, ignore_status=True) self.assertEqual(p.command, expected_command) + @unittest.skipUnless(TRUE_CMD, + '"true" binary not available') @mock.patch.object(path, 'find_command', mock.Mock(return_value=TRUE_CMD)) @mock.patch.object(os, 'getuid', mock.Mock(return_value=0)) @@ -195,6 +225,8 @@ class TestProcessRun(unittest.TestCase): p = process.run(cmd='ls -l', sudo=True, ignore_status=True) self.assertEqual(p.command, expected_command) + @unittest.skipUnless(TRUE_CMD, + '"true" binary not available') @mock.patch.object(path, 'find_command', mock.Mock(return_value=TRUE_CMD)) @mock.patch.object(os, 'getuid', mock.Mock(return_value=1000)) @@ -210,6 +242,8 @@ class TestProcessRun(unittest.TestCase): p = process.run(cmd='ls -l', sudo=True, shell=True, ignore_status=True) self.assertEqual(p.command, expected_command) + @unittest.skipUnless(TRUE_CMD, + '"true" binary not available') @mock.patch.object(path, 'find_command', mock.Mock(return_value=TRUE_CMD)) @mock.patch.object(os, 'getuid', mock.Mock(return_value=0)) @@ -218,6 +252,16 @@ class TestProcessRun(unittest.TestCase): p = process.run(cmd='ls -l', sudo=True, shell=True, ignore_status=True) self.assertEqual(p.command, expected_command) + @unittest.skipUnless(ECHO_CMD, "Echo command not available in system") + def test_run_unicode_output(self): + # Using encoded string as shlex does not support decoding + # but the behavior is exactly the same as if shell binary + # produced unicode + text = u"Avok\xe1do" + result = process.run("%s %s" % (ECHO_CMD, text), encoding='utf-8') + self.assertEqual(result.stdout, text.encode('utf-8') + b'\n') + self.assertEqual(result.stdout_text, text + '\n') + class MiscProcessTests(unittest.TestCase):