未验证 提交 651dab4a 编写于 作者: H hong 提交者: GitHub

Catch exception in download (#28090)

* cat exeception in request download; test=develop

* add re-try in download; test=develop
上级 8327accc
......@@ -88,27 +88,31 @@ def download(url, module_name, md5sum, save_name=None):
sys.stderr.write("Cache file %s not found, downloading %s \n" %
(filename, url))
sys.stderr.write("Begin to download\n")
r = requests.get(url, stream=True)
total_length = r.headers.get('content-length')
if total_length is None:
with open(filename, 'wb') as f:
shutil.copyfileobj(r.raw, f)
else:
with open(filename, 'wb') as f:
chunk_size = 4096
total_length = int(total_length)
total_iter = total_length / chunk_size + 1
log_interval = total_iter / 20 if total_iter > 20 else 1
log_index = 0
for data in r.iter_content(chunk_size=chunk_size):
if six.PY2:
data = six.b(data)
f.write(data)
log_index += 1
if log_index % log_interval == 0:
sys.stderr.write(".")
sys.stdout.flush()
try:
r = requests.get(url, stream=True)
total_length = r.headers.get('content-length')
if total_length is None:
with open(filename, 'wb') as f:
shutil.copyfileobj(r.raw, f)
else:
with open(filename, 'wb') as f:
chunk_size = 4096
total_length = int(total_length)
total_iter = total_length / chunk_size + 1
log_interval = total_iter / 20 if total_iter > 20 else 1
log_index = 0
for data in r.iter_content(chunk_size=chunk_size):
if six.PY2:
data = six.b(data)
f.write(data)
log_index += 1
if log_index % log_interval == 0:
sys.stderr.write(".")
sys.stdout.flush()
except Exception as e:
# re-try
continue
sys.stderr.write("\nDownload finished\n")
sys.stdout.flush()
return filename
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册