未验证 提交 5e5b0bfb 编写于 作者: W wangxinxin08 提交者: GitHub

fix mkdirs to avoid race problem (#6975)

上级 21dfe9cf
...@@ -29,6 +29,7 @@ import base64 ...@@ -29,6 +29,7 @@ import base64
import binascii import binascii
import tarfile import tarfile
import zipfile import zipfile
import errno
from paddle.utils.download import _get_unique_endpoints from paddle.utils.download import _get_unique_endpoints
from ppdet.core.workspace import BASE_KEY from ppdet.core.workspace import BASE_KEY
...@@ -110,6 +111,20 @@ DOWNLOAD_RETRY_LIMIT = 3 ...@@ -110,6 +111,20 @@ DOWNLOAD_RETRY_LIMIT = 3
PPDET_WEIGHTS_DOWNLOAD_URL_PREFIX = 'https://paddledet.bj.bcebos.com/' PPDET_WEIGHTS_DOWNLOAD_URL_PREFIX = 'https://paddledet.bj.bcebos.com/'
# When running unit tests, there could be multiple processes that
# trying to create DATA_HOME directory simultaneously, so we cannot
# use a if condition to check for the existence of the directory;
# instead, we use the filesystem as the synchronization mechanism by
# catching returned errors.
def must_mkdirs(path):
try:
os.makedirs(path)
except OSError as exc:
if exc.errno != errno.EEXIST:
raise
pass
def parse_url(url): def parse_url(url):
url = url.replace("ppdet://", PPDET_WEIGHTS_DOWNLOAD_URL_PREFIX) url = url.replace("ppdet://", PPDET_WEIGHTS_DOWNLOAD_URL_PREFIX)
return url return url
...@@ -344,8 +359,7 @@ def _download(url, path, md5sum=None): ...@@ -344,8 +359,7 @@ def _download(url, path, md5sum=None):
url (str): download url url (str): download url
path (str): download to given path path (str): download to given path
""" """
if not osp.exists(path): must_mkdirs(path)
os.makedirs(path)
fname = osp.split(url)[-1] fname = osp.split(url)[-1]
fullname = osp.join(path, fname) fullname = osp.join(path, fname)
...@@ -407,8 +421,7 @@ def _download_dist(url, path, md5sum=None): ...@@ -407,8 +421,7 @@ def _download_dist(url, path, md5sum=None):
fullname = osp.join(path, fname) fullname = osp.join(path, fname)
lock_path = fullname + '.download.lock' lock_path = fullname + '.download.lock'
if not osp.isdir(path): must_mkdirs(path)
os.makedirs(path)
if not osp.exists(fullname): if not osp.exists(fullname):
with open(lock_path, 'w'): # touch with open(lock_path, 'w'): # touch
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册