From 8072895cccb45c88065058a66912a46bbba5e3d8 Mon Sep 17 00:00:00 2001 From: Kaipeng Deng Date: Mon, 28 Sep 2020 13:59:47 +0800 Subject: [PATCH] check md5 after weights download (#1312) * check md5 after weights download --- ppdet/utils/download.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/ppdet/utils/download.py b/ppdet/utils/download.py index cc78e4753..2ddb8be15 100644 --- a/ppdet/utils/download.py +++ b/ppdet/utils/download.py @@ -22,6 +22,8 @@ import shutil import requests import tqdm import hashlib +import binascii +import base64 import tarfile import zipfile @@ -304,9 +306,20 @@ def _download(url, path, md5sum=None): for chunk in req.iter_content(chunk_size=1024): if chunk: f.write(chunk) - shutil.move(tmp_fullname, fullname) - return fullname + # check md5 after download in Content-MD5 in req.headers + content_md5 = req.headers.get('content-md5') + if not content_md5 or _md5check( + tmp_fullname, + binascii.hexlify(base64.b64decode(content_md5.strip( + '"'))).decode()): + shutil.move(tmp_fullname, fullname) + return fullname + else: + logger.warn( + "Download from url imcomplete, try downloading again...") + os.remove(tmp_fullname) + continue def _md5check(fullname, md5sum=None): -- GitLab