未验证 提交 651dab4a 编写于 作者: H hong 提交者: GitHub

Catch exception in download (#28090)

* cat exeception in request download; test=develop

* add re-try in download; test=develop
上级 8327accc
...@@ -88,27 +88,31 @@ def download(url, module_name, md5sum, save_name=None): ...@@ -88,27 +88,31 @@ def download(url, module_name, md5sum, save_name=None):
sys.stderr.write("Cache file %s not found, downloading %s \n" % sys.stderr.write("Cache file %s not found, downloading %s \n" %
(filename, url)) (filename, url))
sys.stderr.write("Begin to download\n") sys.stderr.write("Begin to download\n")
r = requests.get(url, stream=True) try:
total_length = r.headers.get('content-length') r = requests.get(url, stream=True)
total_length = r.headers.get('content-length')
if total_length is None:
with open(filename, 'wb') as f: if total_length is None:
shutil.copyfileobj(r.raw, f) with open(filename, 'wb') as f:
else: shutil.copyfileobj(r.raw, f)
with open(filename, 'wb') as f: else:
chunk_size = 4096 with open(filename, 'wb') as f:
total_length = int(total_length) chunk_size = 4096
total_iter = total_length / chunk_size + 1 total_length = int(total_length)
log_interval = total_iter / 20 if total_iter > 20 else 1 total_iter = total_length / chunk_size + 1
log_index = 0 log_interval = total_iter / 20 if total_iter > 20 else 1
for data in r.iter_content(chunk_size=chunk_size): log_index = 0
if six.PY2: for data in r.iter_content(chunk_size=chunk_size):
data = six.b(data) if six.PY2:
f.write(data) data = six.b(data)
log_index += 1 f.write(data)
if log_index % log_interval == 0: log_index += 1
sys.stderr.write(".") if log_index % log_interval == 0:
sys.stdout.flush() sys.stderr.write(".")
sys.stdout.flush()
except Exception as e:
# re-try
continue
sys.stderr.write("\nDownload finished\n") sys.stderr.write("\nDownload finished\n")
sys.stdout.flush() sys.stdout.flush()
return filename return filename
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册