diff --git a/avocado/core/data_dir.py b/avocado/core/data_dir.py index 18e62b9891df0eb5a1363b031ae8f3d5b2be333f..1f60bc6edca4817993104172f488e3f6f8015215 100755 --- a/avocado/core/data_dir.py +++ b/avocado/core/data_dir.py @@ -33,6 +33,7 @@ import shutil import time import tempfile +from avocado.utils import path from avocado.settings import settings _BASE_DIR = os.path.join(sys.modules[__name__].__file__, "..", "..", "..") @@ -204,9 +205,7 @@ def get_job_logs_dir(args=None): else: logdir = get_logs_dir() debugbase = 'run-%s' % start_time - debugdir = os.path.join(logdir, debugbase) - if not os.path.isdir(debugdir): - os.makedirs(debugdir) + debugdir = path.init_dir(logdir, debugbase) latestdir = os.path.join(logdir, "latest") try: os.unlink(latestdir) diff --git a/avocado/job.py b/avocado/job.py index f9df91c41eb173a6627103592891b65ebc609dbb..15fae08d64860e335f7f0da280e7488ba4e8cb96 100644 --- a/avocado/job.py +++ b/avocado/job.py @@ -34,6 +34,7 @@ from avocado.core import status from avocado.core import exceptions from avocado.core import error_codes from avocado.utils import archive +from avocado.utils import path from avocado import multiplex_config from avocado import test from avocado import result @@ -149,7 +150,7 @@ class TestRunner(object): # If there's nothing inside the queue after timeout, the process # must be terminated. send_signal(p, signal.SIGUSR1) - test_instance = q.get(timeout=0.1) + test_instance = q.get() self.result.check_test(test_instance) if not status.mapping[test_instance.status]: @@ -191,9 +192,7 @@ class Job(object): self.test_index = 1 self.status = "RUNNING" self.result_proxy = result.TestResultProxy() - self.sysinfo_dir = os.path.join(self.logdir, 'sysinfo') - if not os.path.isdir(self.sysinfo_dir): - os.makedirs(self.sysinfo_dir) + self.sysinfo_dir = path.init_dir(self.logdir, 'sysinfo') self.sysinfo_logger = sysinfo.SysInfo(basedir=self.sysinfo_dir) self.output_manager = output.OutputManager() diff --git a/avocado/plugins/vm.py b/avocado/plugins/vm.py index 5ef272df490eb44b35e4b66a6a5c6002657a9227..085149d74fdcb996c7a507c3ffa7a2cbb0d30c3e 100644 --- a/avocado/plugins/vm.py +++ b/avocado/plugins/vm.py @@ -26,7 +26,6 @@ from avocado.result import TestResult from avocado.plugins import plugin from avocado.utils import virt from avocado.utils import archive -from avocado.utils.misc import unique class Test(object): @@ -119,7 +118,7 @@ class VMTestResult(TestResult): def _copy_tests(self): self.vm.remote.makedir(self.remote_test_dir) - uniq_urls = unique(self.urls) + uniq_urls = list(set(self.urls)) for url in uniq_urls: test_path = os.path.join(self.test_dir, url) self.vm.remote.send_files(test_path, self.remote_test_dir) diff --git a/avocado/sysinfo.py b/avocado/sysinfo.py index 9e881d23c88cbf3857acc6b73351ec1a2f009df5..8166a4e9140e456ccc397aa4f218a536277d076b 100644 --- a/avocado/sysinfo.py +++ b/avocado/sysinfo.py @@ -100,7 +100,7 @@ class Loggable(object): """ path = os.path.join(logdir, self.logf) if os.path.exists(path): - return utils.misc.read_one_line(path) + return utils.io.read_one_line(path) else: return "" @@ -329,10 +329,7 @@ class SysInfo(object): logging packages is a costly operation). """ if basedir is None: - basedir = os.path.join(os.getcwd(), 'sysinfo') - - if not os.path.isdir(basedir): - os.makedirs(basedir) + basedir = utils.path.init_dir(os.getcwd(), 'sysinfo') self.basedir = basedir self.log_packages = log_packages @@ -465,7 +462,7 @@ class SysInfo(object): def _log_installed_packages(self, path): installed_path = os.path.join(path, "installed_packages") installed_packages = "\n".join(self._get_installed_packages()) + "\n" - utils.misc.write_file(installed_path, installed_packages) + utils.io.write_file(installed_path, installed_packages) def _log_modified_packages(self, path): """ @@ -475,18 +472,16 @@ class SysInfo(object): new_packages = set(self._get_installed_packages()) added_path = os.path.join(path, "added_packages") added_packages = "\n".join(new_packages - old_packages) + "\n" - utils.misc.write_file(added_path, added_packages) + utils.io.write_file(added_path, added_packages) removed_path = os.path.join(self.basedir, "removed_packages") removed_packages = "\n".join(old_packages - new_packages) + "\n" - utils.misc.write_file(removed_path, removed_packages) + utils.io.write_file(removed_path, removed_packages) def start_job_hook(self): """ Logging hook called whenever a job starts, and again after reboot. """ - pre_dir = os.path.join(self.basedir, 'pre') - if not os.path.isdir(pre_dir): - os.makedirs(pre_dir) + pre_dir = utils.path.init_dir(self.basedir, 'pre') for log in self.start_job_loggables: log.run(pre_dir) @@ -497,9 +492,7 @@ class SysInfo(object): """ Logging hook called whenever a job starts, and again after reboot. """ - post_dir = os.path.join(self.basedir, 'post') - if not os.path.isdir(post_dir): - os.makedirs(post_dir) + post_dir = utils.path.init_dir(self.basedir, 'post') for log in self.start_job_loggables: log.run(post_dir) @@ -510,9 +503,7 @@ class SysInfo(object): """ Logging hook called before a test starts. """ - pre_dir = os.path.join(self.basedir, 'pre') - if not os.path.isdir(pre_dir): - os.makedirs(pre_dir) + pre_dir = utils.path.init_dir(self.basedir, 'pre') for log in self.start_test_loggables: log.run(pre_dir) @@ -523,9 +514,7 @@ class SysInfo(object): """ Logging hook called after a test finishes. """ - post_dir = os.path.join(self.basedir, 'post') - if not os.path.isdir(post_dir): - os.makedirs(post_dir) + post_dir = utils.path.init_dir(self.basedir, 'post') for log in self.end_test_loggables: log.run(post_dir) @@ -536,9 +525,7 @@ class SysInfo(object): """ Logging hook called before a test iteration """ - pre_dir = os.path.join(self.basedir, 'pre') - if not os.path.isdir(pre_dir): - os.makedirs(pre_dir) + pre_dir = utils.path.init_dir(self.basedir, 'pre') for log in self.start_iteration_loggables: log.run(pre_dir) @@ -546,10 +533,7 @@ class SysInfo(object): """ Logging hook called after a test iteration """ - post_dir = os.path.join(self.basedir, 'post') - if not os.path.isdir(post_dir): - os.makedirs(post_dir) - post_dir = os.path.join(self.basedir, 'post') + post_dir = utils.path.init_dir(self.basedir, 'post') for log in self.end_iteration_loggables: log.run(post_dir) diff --git a/avocado/test.py b/avocado/test.py index 822cb00d7aa1899a3246901a9c919ce7b1401497..e378fca4cc85d7fdc460a55f70c7780305781b34 100644 --- a/avocado/test.py +++ b/avocado/test.py @@ -26,6 +26,7 @@ import unittest from avocado.core import data_dir from avocado.core import exceptions +from avocado.utils import path from avocado.utils import process from avocado.utils.params import Params from avocado import sysinfo @@ -116,23 +117,15 @@ class Test(unittest.TestCase): self.job = job self.basedir = os.path.join(data_dir.get_test_dir(), self.name) self.datadir = os.path.join(self.basedir, 'data') - self.workdir = os.path.join(data_dir.get_tmp_dir(), self.name) - if not os.path.isdir(self.workdir): - os.makedirs(self.workdir) - self.srcdir = os.path.join(self.workdir, 'src') - if not os.path.isdir(self.srcdir): - os.makedirs(self.srcdir) + self.workdir = path.init_dir(data_dir.get_tmp_dir(), self.name) + self.srcdir = path.init_dir(self.workdir, 'src') if base_logdir is None: base_logdir = data_dir.get_job_logs_dir() self.tagged_name = self.get_tagged_name(base_logdir) - self.logdir = os.path.join(base_logdir, self.tagged_name) - if not os.path.isdir(self.logdir): - os.makedirs(self.logdir) + self.logdir = path.init_dir(base_logdir, self.tagged_name) self.logfile = os.path.join(self.logdir, 'debug.log') - self.outputdir = os.path.join(self.logdir, 'data') - if not os.path.isdir(self.outputdir): - os.makedirs(self.outputdir) - self.sysinfodir = os.path.join(self.logdir, 'sysinfo') + self.outputdir = path.init_dir(self.logdir, 'data') + self.sysinfodir = path.init_dir(self.logdir, 'sysinfo') self.sysinfo_logger = sysinfo.SysInfo(basedir=self.sysinfodir) self.log = logging.getLogger("avocado.test") diff --git a/avocado/utils/__init__.py b/avocado/utils/__init__.py index ce702b06f61209845c56c4c2b210d3e636f78968..fd684f8bb9d0df16830c3e9cb8028ae46ddb4756 100644 --- a/avocado/utils/__init__.py +++ b/avocado/utils/__init__.py @@ -12,9 +12,14 @@ # Copyright: Red Hat Inc. 2013-2014 # Author: Lucas Meneghel Rodrigues -import logging -import random -import string +import archive +import build +import crypto import download +import io +import memory +import network +import params import process -import misc +import remote +import virt diff --git a/avocado/utils/misc.py b/avocado/utils/io.py similarity index 53% rename from avocado/utils/misc.py rename to avocado/utils/io.py index f19b7f7384107ca97fe8e85c3c171761380e18d1..ffdf5b010688d412099cc579c10dfec56c03a3b7 100644 --- a/avocado/utils/misc.py +++ b/avocado/utils/io.py @@ -12,18 +12,26 @@ # client/shared/utils.py # Authors: Martin J Bligh , Andy Whitcroft +""" +Avocado generic IO related functions. +""" + import logging -import os log = logging.getLogger('avocado.test') def ask(question, auto=False): """ - Raw input with a prompt. + Prompt the user with a (y/n) question. :param question: Question to be asked + :type question: str :param auto: Whether to return "y" instead of asking the question + :type auto: bool + + :return: User answer + :rtype: str """ if auto: log.info("%s (y/n) y" % question) @@ -36,92 +44,51 @@ def read_file(filename): Read the entire contents of file. :param filename: Path to the file. + :type filename: str + + :return: File contents + :rtype: str """ with open(filename, 'r') as file_obj: contents = file_obj.read() return contents - f = open(filename) - try: - return f.read() - finally: - f.close() - def read_one_line(filename): """ Read the first line of filename. :param filename: Path to the file. + :type filename: str + + :return: First line contents + :rtype: str """ with open(filename, 'r') as file_obj: line = file_obj.readline().rstrip('\n') return line -def write_one_line(filename, line): - """ - Write one line of text to filename. - - :param filename: Path to the file. - :param line: Line to be written. - """ - write_file(filename, line.rstrip('\n') + '\n') - - def write_file(filename, data): """ Write data to a file. :param filename: Path to the file. + :type filename: str :param line: Line to be written. + :type line: str """ with open(filename, 'w') as file_obj: file_obj.write(data) -def get_relative_path(path, reference): - """ - Given 2 absolute paths "path" and "reference", compute the path of - "path" as relative to the directory "reference". - - :param path: The absolute path to convert to a relative path. - :param reference: An absolute directory path to which the relative - path will be computed. - """ - # normalize the paths (remove double slashes, etc) - assert(os.path.isabs(path)) - assert(os.path.isabs(reference)) - - path = os.path.normpath(path) - reference = os.path.normpath(reference) - - # we could use os.path.split() but it splits from the end - path_list = path.split(os.path.sep)[1:] - ref_list = reference.split(os.path.sep)[1:] - - # find the longest leading common path - for i in xrange(min(len(path_list), len(ref_list))): - if path_list[i] != ref_list[i]: - # decrement i so when exiting this loop either by no match or by - # end of range we are one step behind - i -= 1 - break - i += 1 - # drop the common part of the paths, not interested in that anymore - del path_list[:i] - - # for each uncommon component in the reference prepend a ".." - path_list[:0] = ['..'] * (len(ref_list) - i) - - return os.path.join(*path_list) - - -def unique(lst): +def write_one_line(filename, line): """ - Return a list of the elements in list, but without duplicates. + Write one line of text to filename. - :param lst: List with values. - :return: List with non duplicate elements. + :param filename: Path to the file. + :type filename: str + :param line: Line to be written. + :type line: str """ - return list(set(lst)) + write_file(filename, line.rstrip('\n') + '\n') diff --git a/avocado/utils/path.py b/avocado/utils/path.py new file mode 100644 index 0000000000000000000000000000000000000000..d175265a2354557e1961b98e631ba501479f5172 --- /dev/null +++ b/avocado/utils/path.py @@ -0,0 +1,34 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See LICENSE for more details. +# +# Copyright: Red Hat Inc. 2013-2014 +# Author: Yiqiao Pu + +""" +Avocado path related functions. +""" + +import os + + +def init_dir(*args): + """ + Wrapper around os.path.join that creates dirs based on the final path. + + :param args: List of dir arguments that will be os.path.joined. + :type directory: list + :return: directory. + :rtype: str + """ + directory = os.path.join(*args) + if not os.path.isdir(directory): + os.makedirs(directory) + return directory diff --git a/avocado/utils/process.py b/avocado/utils/process.py index 04242acf6f4740a15da84f58ac78bc5baac60dc0..545accf7b9fd20e58e3900faa49838f341f2a727 100644 --- a/avocado/utils/process.py +++ b/avocado/utils/process.py @@ -26,7 +26,6 @@ import time import threading from avocado.core import exceptions -from avocado.utils import misc log = logging.getLogger('avocado.test') @@ -64,7 +63,7 @@ def find_command(cmd): path_paths = os.environ['PATH'].split(":") except IndexError: path_paths = [] - path_paths = misc.unique(common_bin_paths + path_paths) + path_paths = list(set(common_bin_paths + path_paths)) for dir_path in path_paths: cmd_path = os.path.join(dir_path, cmd)