fix mkdirs to avoid race problem (#6975)

ppdet/utils/download.py

@@ -29,6 +29,7 @@ import base64
 import binascii
 import tarfile
 import zipfile
+import errno
 
 from paddle.utils.download import _get_unique_endpoints
 from ppdet.core.workspace import BASE_KEY
@@ -110,6 +111,20 @@ DOWNLOAD_RETRY_LIMIT = 3
 PPDET_WEIGHTS_DOWNLOAD_URL_PREFIX = 'https://paddledet.bj.bcebos.com/'
 
 
+# When running unit tests, there could be multiple processes that
+# trying to create DATA_HOME directory simultaneously, so we cannot
+# use a if condition to check for the existence of the directory;
+# instead, we use the filesystem as the synchronization mechanism by
+# catching returned errors.
+def must_mkdirs(path):
+    try:
+        os.makedirs(path)
+    except OSError as exc:
+        if exc.errno != errno.EEXIST:
+            raise
+        pass
+
+
 def parse_url(url):
     url = url.replace("ppdet://", PPDET_WEIGHTS_DOWNLOAD_URL_PREFIX)
     return url
@@ -344,8 +359,7 @@ def _download(url, path, md5sum=None):
     url (str): download url
     path (str): download to given path
     """
-    if not osp.exists(path):
-        os.makedirs(path)
+    must_mkdirs(path)
 
     fname = osp.split(url)[-1]
     fullname = osp.join(path, fname)
@@ -407,8 +421,7 @@ def _download_dist(url, path, md5sum=None):
             fullname = osp.join(path, fname)
             lock_path = fullname + '.download.lock'
 
-            if not osp.isdir(path):
-                os.makedirs(path)
+            must_mkdirs(path)
 
             if not osp.exists(fullname):
                 with open(lock_path, 'w'):  # touch