Merging pull request 1278

* https://github.com/avocado-framework/avocado:
  Add expire time for asset fetcher
  Move _validate_job_timeout() to utils

avocado/core/test.py

@@ -625,7 +625,7 @@ class Test(unittest.TestCase):
         raise exceptions.TestSkipError(message)
 
     def fetch_asset(self, name, asset_hash=None, algorithm='sha1',
-                    locations=None):
+                    locations=None, expire=None):
         """
         Method o call the utils.asset in order to fetch and asset file
         supporting hash check, caching and multiple locations.
@@ -635,10 +635,13 @@ class Test(unittest.TestCase):
         :param algorithm: hash algorithm (optional, defaults to sha1)
         :param locations: list of URLs from where the asset can be
                           fetched (optional)
+        :param expire: time for the asset to expire
         :returns: asset file local path
         """
+        if expire is not None:
+            expire = data_structures.time_to_seconds(str(expire))
         return asset.Asset(name, asset_hash, algorithm, locations,
-                           self.cache_dirs).fetch()
+                           self.cache_dirs, expire).fetch()
 
 
 class SimpleTest(Test):

avocado/plugins/run.py

@@ -26,6 +26,7 @@ from avocado.core import loader
 from avocado.core import multiplexer
 from avocado.core.plugin_interfaces import CLICmd
 from avocado.core.settings import settings
+from avocado.utils.data_structures import time_to_seconds
 
 
 class Run(CLICmd):
@@ -159,34 +160,13 @@ class Run(CLICmd):
                 node = args.default_avocado_params.get_node(value[0], True)
                 node.value[value[1]] = value[2]
 
-    def _validate_job_timeout(self, raw_timeout):
-        units = {'s': 1, 'm': 60, 'h': 3600, 'd': 86400}
-        mult = 1
-        if raw_timeout is not None:
-            try:
-                unit = raw_timeout[-1].lower()
-                if unit in units:
-                    mult = units[unit]
-                    timeout = int(raw_timeout[:-1]) * mult
-                else:
-                    timeout = int(raw_timeout)
-                if timeout < 1:
-                    raise ValueError()
-            except (ValueError, TypeError):
-                log = logging.getLogger("avocado.app")
-                log.error("Invalid number '%s' for job timeout. Use an "
-                          "integer number greater than 0", raw_timeout)
-                sys.exit(exit_codes.AVOCADO_FAIL)
-        else:
-            timeout = 0
-        return timeout
-
     def run(self, args):
         """
         Run test modules or simple tests.
 
         :param args: Command line args received from the run subparser.
         """
+        log = logging.getLogger("avocado.app")
         self._activate(args)
         if args.unique_job_id is not None:
             try:
@@ -194,9 +174,12 @@ class Run(CLICmd):
                 if len(args.unique_job_id) != 40:
                     raise ValueError
             except ValueError:
-                log = logging.getLogger("avocado.app")
                 log.error('Unique Job ID needs to be a 40 digit hex number')
                 sys.exit(exit_codes.AVOCADO_FAIL)
-        args.job_timeout = self._validate_job_timeout(args.job_timeout)
+        try:
+            args.job_timeout = time_to_seconds(args.job_timeout)
+        except ValueError as e:
+            log.error(e.message)
+            sys.exit(exit_codes.AVOCADO_FAIL)
         job_instance = job.Job(args)
         return job_instance.run()

avocado/utils/asset.py

@@ -16,9 +16,12 @@
 Asset fetcher from multiple locationss
 """
 
+import errno
 import logging
 import os
 import re
+import stat
+import time
 import urlparse
 
 from . import crypto
@@ -34,7 +37,8 @@ class Asset(object):
     Try to fetch/verify an asset file from multiple locations.
     """
 
-    def __init__(self, name, asset_hash, algorithm, locations, cache_dirs):
+    def __init__(self, name, asset_hash, algorithm, locations, cache_dirs,
+                 expire):
         """
         Initialize the Asset() and fetches the asset file. The path for
         the fetched file can be reached using the self.path attribute.
@@ -44,6 +48,7 @@ class Asset(object):
         :param algorithm: hash algorithm
         :param locations: list of locations fetch asset from
         :params cache_dirs: list of cache directories
+        :params expire: time in seconds for the asset to expire
         """
         self.name = name
         self.asset_hash = asset_hash
@@ -52,6 +57,7 @@ class Asset(object):
         self.cache_dirs = cache_dirs
         self.nameobj = urlparse.urlparse(self.name)
         self.basename = os.path.basename(self.nameobj.path)
+        self.expire = expire
 
     def fetch(self):
         urls = []
@@ -64,7 +70,9 @@ class Asset(object):
         for cache_dir in self.cache_dirs:
             cache_dir = os.path.expanduser(cache_dir)
             self.asset_file = os.path.join(cache_dir, self.basename)
-            if self._check_file(self.asset_file, self.asset_hash, self.algorithm):
+            if (self._check_file(self.asset_file,
+               self.asset_hash, self.algorithm) and not
+               self._is_expired(self.asset_file, self.expire)):
                 return self.asset_file
 
         # If we get to this point, file is not in any cache directory
@@ -117,9 +125,15 @@ class Asset(object):
                         path = urlobj.path
                     log.debug('Looking for file on %s.' % path)
                     if self._check_file(path):
-                        os.symlink(path, self.asset_file)
+                        try:
+                            os.symlink(path, self.asset_file)
+                        except OSError as e:
+                            if e.errno == errno.EEXIST:
+                                os.remove(self.asset_file)
+                                os.symlink(path, self.asset_file)
                         log.debug('Symlink created %s -> %s.' %
                                   (self.asset_file, path))
+
                     else:
                         continue
                     if self._check_file(self.asset_file, self.asset_hash,
@@ -178,3 +192,13 @@ class Asset(object):
             log.error('Asset %s corrupted (hash expected:%s, hash found:%s).' %
                       (path, filehash, discovered_hash))
             return False
+
+    @staticmethod
+    def _is_expired(path, expire):
+        if expire is None:
+            return False
+        creation_time = os.lstat(path)[stat.ST_CTIME]
+        expire_time = creation_time + expire
+        if time.time() > expire_time:
+            return True
+        return False

avocado/utils/data_structures.py

@@ -218,3 +218,26 @@ class CallbackRegister(object):
                   to be executed!
         """
         self.run()
+
+
+def time_to_seconds(time):
+    """
+    Convert time in minutes, hours and days to seconds.
+    :param time: Time, optionally including the unit (i.e. '10d')
+    """
+    units = {'s': 1, 'm': 60, 'h': 3600, 'd': 86400}
+    if time is not None:
+        try:
+            unit = time[-1].lower()
+            if unit in units:
+                mult = units[unit]
+                seconds = int(time[:-1]) * mult
+            else:
+                seconds = int(time)
+        except (ValueError, TypeError) as e:
+            raise ValueError("Invalid value '%s' for time. Use a string with "
+                             "the number and optionally the time unit (s, m, "
+                             "h or d)." % time)
+    else:
+        seconds = 0
+    return seconds

docs/source/WritingTests.rst

@@ -538,6 +538,11 @@ Detailing the ``fetch_asset()`` attributes:
   file, including the file name. The first success will skip the next
   locations. Notice that for ``file://`` we just create a symbolic link in the
   cache directory, pointing to the file original location.
+* ``expire:`` (optional) time period that the cached file will be considered
+  valid. After that period, the file will be dowloaded again. The value can
+  be an integer or a string containig the time and the unit. Example: '10d'
+  (ten days). Valid units are ``s`` (second), ``m`` (minute), ``h`` (hour) and
+  ``d`` (day).
 
 The expected ``return`` is the asset file path or an exception.
 

selftests/functional/test_job_timeout.py

@@ -135,15 +135,15 @@ class JobTimeOutTest(unittest.TestCase):
 
     def test_invalid_values(self):
         cmd_line = ('./scripts/avocado run --job-results-dir %s --sysinfo=off '
-                    '--job-timeout=0 examples/tests/passtest.py' % self.tmpdir)
+                    '--job-timeout=1,5 examples/tests/passtest.py' % self.tmpdir)
         result = process.run(cmd_line, ignore_status=True)
         self.assertEqual(result.exit_status, exit_codes.AVOCADO_FAIL)
-        self.assertIn('Invalid number', result.stderr)
+        self.assertIn('Invalid value', result.stderr)
         cmd_line = ('./scripts/avocado run --job-results-dir %s --sysinfo=off '
                     '--job-timeout=123x examples/tests/passtest.py' % self.tmpdir)
         result = process.run(cmd_line, ignore_status=True)
         self.assertEqual(result.exit_status, exit_codes.AVOCADO_FAIL)
-        self.assertIn('Invalid number', result.stderr)
+        self.assertIn('Invalid value', result.stderr)
 
     def test_valid_values(self):
         cmd_line = ('./scripts/avocado run --job-results-dir %s --sysinfo=off '

selftests/unit/test_utils_asset.py

@@ -24,7 +24,8 @@ class TestAsset(unittest.TestCase):
                                   asset_hash=self.assethash,
                                   algorithm='sha1',
                                   locations=None,
-                                  cache_dirs=[self.cache_dir]).fetch()
+                                  cache_dirs=[self.cache_dir],
+                                  expire=None).fetch()
         expected_tarball = os.path.join(self.cache_dir, self.assetname)
         self.assertEqual(foo_tarball, expected_tarball)
         hashfile = '.'.join([expected_tarball, 'sha1'])
@@ -39,7 +40,8 @@ class TestAsset(unittest.TestCase):
                                   asset_hash=self.assethash,
                                   algorithm='sha1',
                                   locations=[self.url],
-                                  cache_dirs=[self.cache_dir]).fetch()
+                                  cache_dirs=[self.cache_dir],
+                                  expire=None).fetch()
         expected_tarball = os.path.join(self.cache_dir, self.assetname)
         self.assertEqual(foo_tarball, expected_tarball)
         hashfile = '.'.join([expected_tarball, 'sha1'])
@@ -49,9 +51,49 @@ class TestAsset(unittest.TestCase):
             content = f.read()
         self.assertEqual(content, expected_content)
 
+    def testFecth_expire(self):
+        foo_tarball = asset.Asset(self.assetname,
+                                  asset_hash=self.assethash,
+                                  algorithm='sha1',
+                                  locations=[self.url],
+                                  cache_dirs=[self.cache_dir],
+                                  expire=None).fetch()
+        with open(foo_tarball, 'r') as f:
+            content1 = f.read()
+
+        # Create the file in a diferent location with a diferent content
+        new_assetdir = tempfile.mkdtemp(dir=self.basedir)
+        new_localpath = os.path.join(new_assetdir, self.assetname)
+        new_url = 'file://%s' % new_localpath
+        with open(new_localpath, 'w') as f:
+            f.write('Changed!')
+
+        # Dont expire cached file
+        asset.Asset(self.assetname,
+                    asset_hash=self.assethash,
+                    algorithm='sha1',
+                    locations=[new_url],
+                    cache_dirs=[self.cache_dir],
+                    expire=None).fetch()
+        with open(foo_tarball, 'r') as f:
+            content2 = f.read()
+        self.assertEqual(content1, content2)
+
+        # Expire cached file
+        asset.Asset(self.assetname,
+                    asset_hash=self.assethash,
+                    algorithm='sha1',
+                    locations=[new_url],
+                    cache_dirs=[self.cache_dir],
+                    expire=-1).fetch()
+        with open(foo_tarball, 'r') as f:
+            content2 = f.read()
+        self.assertNotEqual(content1, content2)
+
     def testException(self):
         a = asset.Asset(name='bar.tgz', asset_hash=None, algorithm=None,
-                        locations=None, cache_dirs=[self.cache_dir])
+                        locations=None, cache_dirs=[self.cache_dir],
+                        expire=None)
         self.assertRaises(EnvironmentError, a.fetch)
 
     def tearDown(self):